From c5a488afc05372dd46e05559ef04ac9969808507 Mon Sep 17 00:00:00 2001 From: jakobrs Date: Thu, 19 Mar 2020 19:15:55 +0100 Subject: [PATCH 001/350] Remove the --delete option for --gc Running `nix-store --gc --delete` will, as of Nix 2.3.3, simply fail because the --delete option conflicts with the --delete operation. $ nix-store --gc --delete error: only one operation may be specified Try 'nix-store --help' for more information. Furthermore, it has been broken since at least Nix 0.16 (which was released sometime in 2010), which means that any scripts which depend on it should have been broken at least nine years ago. This commit simply formally removes the option. There should be no actual difference in behaviour as far as the user is concerned: it errors with the exact same error message. The manual has been edited to remove any references to the (now gone) --delete option. Other information: * Path for Nix 0.16 used: /nix/store/rp3sgmskn0p0pj1ia2qwd5al6f6pinz4-nix-0.16 --- doc/manual/command-ref/nix-store.xml | 17 ++++------------- src/nix-store/nix-store.cc | 1 - 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 113a3c2e4..e8bbd16e8 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -360,7 +360,6 @@ EOF - bytes @@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of - - - This operation performs an actual garbage - collection. All dead paths are removed from the - store. This is the default. - - - By default, all unreachable paths are deleted. The following @@ -444,10 +435,10 @@ and keep-derivations variables in the Nix configuration file. -With , the collector prints the total -number of freed bytes when it finishes (or when it is interrupted). -With , it prints the number of bytes that -would be freed. +By default, the collector prints the total number of freed bytes +when it finishes (or when it is interrupted). With +, it prints the number of bytes that would +be freed. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 45e152c47..c14d900ae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -577,7 +577,6 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; else if (*i == "--max-freed") { long long maxFreed = getIntArg(*i, i, opFlags.end(), true); options.maxFreed = maxFreed >= 0 ? maxFreed : 0; From afb78ebd34bff9a701d70041abc2ff211390584e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:21:56 +0100 Subject: [PATCH 002/350] libstore: disable resolve-system-dependencies hook This is used to determine the dependency tree of impure libraries so nix knows what paths to open in the sandbox. With the less restrictive defaults it isn't needed anymore. --- src/libstore/globals.hh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 782870547..3aa3653f3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -311,12 +311,7 @@ public: Setting printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting preBuildHook{this, -#if __APPLE__ - nixLibexecDir + "/nix/resolve-system-dependencies", -#else - "", -#endif + Setting preBuildHook{this, "", "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; From 7f2df903d91cd21ab05223344ee4dec0a7d52c41 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:31:20 +0100 Subject: [PATCH 003/350] libstore: relax default sandbox-paths on darwin --- src/libstore/globals.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a8945996e..1a2fcbe22 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -20,13 +20,6 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - Settings settings; static GlobalConfig::Register r1(&settings); @@ -68,7 +61,12 @@ Settings::Settings() sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif - allowedImpureHostPrefixes = tokenizeString(DEFAULT_ALLOWED_IMPURE_PREFIXES); + +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib"); + allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); +#endif } void loadConfFile() From f6c122aaeb08cc3d9e89465b440b25c7e0c87d9e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 21:58:45 +0100 Subject: [PATCH 004/350] sandbox: allow pty devices Nix now runs builds with a pseudo-terminal to enable colored build output. --- src/libstore/sandbox-defaults.sb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 0299d1ee4..c09ce1729 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -71,6 +71,12 @@ (literal "/dev/zero") (subpath "/dev/fd")) +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + ; Does nothing, but reduces build noise. (allow file* (literal "/dev/dtracehelper")) From 2e9bc1245c125f96ce53210751940067d4cf3f1c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 20 Mar 2020 22:12:30 +0100 Subject: [PATCH 005/350] sandbox: fix /bin/sh on catalina Sadly 10.15 changed /bin/sh to a shim which executes bash, this means it can't be used anymore without also opening up the sandbox to allow bash. Failed to exec /bin/bash as variant for /bin/sh (1: Operation not permitted). --- src/libstore/globals.cc | 2 +- src/libstore/sandbox-defaults.sb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1a2fcbe22..7e97f3c22 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -64,7 +64,7 @@ Settings::Settings() /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ - sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /private/tmp /private/var/tmp /usr/lib"); + sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif } diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index c09ce1729..351037822 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -91,3 +91,7 @@ (literal "/etc") (literal "/var") (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) From 9450dece24edd34613d887c7c0a1f2e58d86ecb4 Mon Sep 17 00:00:00 2001 From: Philipp Middendorf Date: Sat, 21 Mar 2020 09:31:39 +0100 Subject: [PATCH 006/350] installer: also test for xz to unpack --- scripts/install.in | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/install.in b/scripts/install.in index 4500fd54a..6709f00d4 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -36,6 +36,7 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" +require_util xz "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" From f694f43d7ddd270706e6e3a5f034336a5a83fb86 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 22 Mar 2020 12:25:47 -0600 Subject: [PATCH 007/350] straightforward port of rust mockup code --- src/libutil/error.cc | 137 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/error.hh | 57 ++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 src/libutil/error.cc create mode 100644 src/libutil/error.hh diff --git a/src/libutil/error.cc b/src/libutil/error.cc new file mode 100644 index 000000000..4af4691d4 --- /dev/null +++ b/src/libutil/error.cc @@ -0,0 +1,137 @@ +#include "error.hh" + +#include + +namespace nix { + +using std::cout; +using std::endl; + +// return basic_format? +string showErrLine(ErrLine &errLine) +{ + if (errLine.columnRange.has_value()) + { + return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); + } + else + { + return (format("(%1%)") % errLine.lineNumber).str(); + }; +} + +void print_code_lines(string &prefix, NixCode &nix_code) +{ + if (nix_code.errLine.has_value()) + { + // previous line of code. + if (nix_code.errLine->prevLineOfCode.has_value()) { + cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber - 1) % *nix_code.errLine->prevLineOfCode; + } + + // line of code containing the error. + cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber) % nix_code.errLine->errLineOfCode; + + // error arrows for the column range. + if (nix_code.errLine->columnRange.has_value()) + { + int start = nix_code.errLine->columnRange->start; + std::string spaces; + for (int i = 0; i < start; ++i) + { + spaces.append(" "); + } + + int len = nix_code.errLine->columnRange->len; + std::string arrows; + for (int i = 0; i < len; ++i) + { + arrows.append("^"); + } + + cout << format("%1% |%2%%3%") % prefix % spaces % arrows; + } + + + // next line of code. + if (nix_code.errLine->nextLineOfCode.has_value()) { + cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber + 1) % *nix_code.errLine->nextLineOfCode; + } + + } + else + { + } + +} + +void print_error(ErrorInfo &einfo) +{ + int errwidth = 80; + string prefix = " "; + + string level_string; + switch (einfo.level) + { + case ErrLevel::elError: + { + level_string = "error:"; // TODO make red. + break; + } + case ErrLevel::elWarning: + { + level_string = "warning:"; // TODO make yellow. + break; + } + } + + int ndl = level_string.length() + 3 + einfo.errName.length() + einfo.toolName.length(); + int dashwidth = errwidth - 3 ? 3 : 80 - ndl; + + string dashes; + for (int i = 0; i < dashwidth; ++i) + dashes.append("-"); + + // divider. + cout << format("%1%%2% %3% %4% %5% %6%") + % prefix + % level_string + % "---" + % einfo.errName + % dashes + % einfo.toolName; + + // filename. + if (einfo.nixCode.has_value()) + { + if (einfo.nixCode->nixFile.has_value()) + { + string eline = einfo.nixCode->errLine.has_value() + ? string(" ") + showErrLine(*einfo.nixCode->errLine) + : ""; + + cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline; + cout << prefix << endl; + } + else + { + cout << format("%1%from command line argument") % prefix << endl; + cout << prefix << endl; + } + } + + // description + cout << prefix << einfo.description << endl; + cout << prefix << endl; + + // lines of code. + if (einfo.nixCode.has_value()) + print_code_lines(prefix, *einfo.nixCode); + + // hint + cout << prefix << einfo.hint << endl; + cout << prefix << endl; + +} + +} diff --git a/src/libutil/error.hh b/src/libutil/error.hh new file mode 100644 index 000000000..0bc1f4e24 --- /dev/null +++ b/src/libutil/error.hh @@ -0,0 +1,57 @@ +#pragma once + +#include "types.hh" +#include +#include +// #include + +using std::string; +using std::optional; +// using boost::format; + +namespace nix { + +enum ErrLevel + { elWarning + , elError + }; + +class ColumnRange { + public: + unsigned int start; + unsigned int len; +}; + +class ErrLine { + public: + int lineNumber; + optional columnRange; + optional prevLineOfCode; + string errLineOfCode; + optional nextLineOfCode; +}; + +class NixCode { + public: + optional nixFile; + optional errLine; +}; + + +class ErrorInfo { + public: + ErrLevel level; + string errName; + string description; + string toolName; + optional nixCode; + string hint; +}; + +string showErrLine(ErrLine &errLine); + +void print_code_lines(string &prefix, NixCode &nix_code); + +void print_error(ErrorInfo &einfo); +} + From 26851dd2c22690838d391ef85b90a99fc00bf9ea Mon Sep 17 00:00:00 2001 From: Greg Price Date: Sat, 21 Mar 2020 22:03:58 -0700 Subject: [PATCH 008/350] installer: Set files read-only when copying into store After installing Nix, I found that all the files and directories initially copied into the store were writable, with mode 644 or 755: drwxr-xr-x 9 root root 4096 Dec 31 1969 /nix/store/ddmmzn4ggz1f66lwxjy64n89864yj9w9-nix-2.3.3 The reason is that that's how they were in the unpacked tarball, and the install-multi-user script used `rsync -p` without doing anything else to affect the permissions. The plain `install` script for a single-user install takes care to do a `chmod -R a-w` on each store path copied. We could do the same here with one more command; or we can pass `--chmod` to rsync, to have it write the files with the desired modes in the first place. Tested the new `rsync` command on both a Linux machine with a reasonably-modern rsync (3.1.3) and a Mac with its default, ancient, rsync 2.6.9, and it works as expected on both. Thankfully the latter is just new enough to have `--chmod`, which dates to rsync 2.6.7. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 13762cba3..35341543e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -567,7 +567,7 @@ install_from_extracted_nix() { cd "$EXTRACTED_NIX_PATH" _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - rsync -rlpt ./store/* "$NIX_ROOT/store/" + rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/" if [ -d "$NIX_INSTALLED_NIX" ]; then echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" From aadd59d005bdabbb801f622dc1a0e3b12a5e5286 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 23 Mar 2020 15:29:49 -0600 Subject: [PATCH 009/350] error test --- src/libutil/error.cc | 36 +++++++++++++++++++++++++----------- src/libutil/error.hh | 2 -- tests/errors/main.cc | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 tests/errors/main.cc diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4af4691d4..71f422622 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -8,7 +8,7 @@ using std::cout; using std::endl; // return basic_format? -string showErrLine(ErrLine &errLine) +string showErrLine(ErrLine &errLine) { if (errLine.columnRange.has_value()) { @@ -22,15 +22,26 @@ string showErrLine(ErrLine &errLine) void print_code_lines(string &prefix, NixCode &nix_code) { + if (nix_code.errLine.has_value()) { // previous line of code. if (nix_code.errLine->prevLineOfCode.has_value()) { - cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber - 1) % *nix_code.errLine->prevLineOfCode; + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nix_code.errLine->lineNumber - 1) + % *nix_code.errLine->prevLineOfCode + << endl; } - // line of code containing the error. - cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber) % nix_code.errLine->errLineOfCode; + // line of code containing the error.%2$+5d% + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nix_code.errLine->lineNumber) + % nix_code.errLine->errLineOfCode + << endl; + + // error arrows for the column range. if (nix_code.errLine->columnRange.has_value()) @@ -49,19 +60,21 @@ void print_code_lines(string &prefix, NixCode &nix_code) arrows.append("^"); } - cout << format("%1% |%2%%3%") % prefix % spaces % arrows; + cout << format("%1% |%2%%3%") % prefix % spaces % arrows; } + // next line of code. if (nix_code.errLine->nextLineOfCode.has_value()) { - cout << format("%1% %2$+5d%| %3%") % prefix % (nix_code.errLine->lineNumber + 1) % *nix_code.errLine->nextLineOfCode; + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nix_code.errLine->lineNumber + 1) + % *nix_code.errLine->nextLineOfCode + << endl; } } - else - { - } } @@ -99,7 +112,8 @@ void print_error(ErrorInfo &einfo) % "---" % einfo.errName % dashes - % einfo.toolName; + % einfo.toolName + << endl; // filename. if (einfo.nixCode.has_value()) @@ -110,7 +124,7 @@ void print_error(ErrorInfo &einfo) ? string(" ") + showErrLine(*einfo.nixCode->errLine) : ""; - cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline; + cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline << endl; cout << prefix << endl; } else diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0bc1f4e24..8dac1508e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -3,11 +3,9 @@ #include "types.hh" #include #include -// #include using std::string; using std::optional; -// using boost::format; namespace nix { diff --git a/tests/errors/main.cc b/tests/errors/main.cc new file mode 100644 index 000000000..1c998103e --- /dev/null +++ b/tests/errors/main.cc @@ -0,0 +1,39 @@ +#include "error.hh" + +#include +#include + +using std::optional; + +int main() { + +using namespace nix; + + ColumnRange columnRange; + columnRange.start = 24; + columnRange.len = 14; + + ErrLine errline; + errline.lineNumber = 7; + errline.columnRange = optional(columnRange); + errline.errLineOfCode = "line of code where the error occurred"; + + NixCode nixcode; + nixcode.nixFile = optional("myfile.nix"); + nixcode.errLine = errline; + + ErrorInfo generic; + generic.level = elError; + generic.errName = "error name"; + generic.description = "general error description"; + generic.toolName = "nixtool.exe"; + generic.nixCode = nixcode; + + print_error(generic); + + return 0; +} + + + + From e40e01c1dd58c12a763d9614ff38dd7a984c9787 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Sat, 21 Mar 2020 22:20:02 -0700 Subject: [PATCH 010/350] doc: Files in the store have modes 444/555, not 644/755 This line has been this way since it was written, in 9e08f5efe in 2006. I think it was just a small mistake then; Eelco's thesis earlier that year says the permission on each file is set to 0444 or 0555 in a derivation's output as part of the build process. In any case I'm pretty sure that's the behavior now. --- doc/manual/command-ref/nix-store.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index e8bbd16e8..1ddb5408d 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1139,7 +1139,7 @@ the information that Nix considers important. For instance, timestamps are elided because all files in the Nix store have their timestamp set to 0 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have -644 or 755 permission. +444 or 555 permission. Also, a NAR archive is canonical, meaning that “equal” paths always produce the same NAR archive. For instance, From c34a20e1f60486140f0ddc5657c077d2fdc9c7f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Nov 2018 19:55:48 +0100 Subject: [PATCH 011/350] EvalState::allocAttr(): Add convenience method (cherry picked from commit c02da997570ac0d9b595d787bea8cb5a4e3cc1f5) --- src/libexpr/attr-set.cc | 6 ++++++ src/libexpr/eval.hh | 1 + 2 files changed, 7 insertions(+) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 0785897d2..b1d61a285 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index eac53b894..5efad8707 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -272,6 +272,7 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); Bindings * allocBindings(size_t capacity); From 5a7e7fc35f934872a9e3e9e9043ff6894b36637a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 15:53:59 +0100 Subject: [PATCH 012/350] Use std::string_view (cherry picked from commit 6529490cc10018d5191e50c482ac1180b96b1a3c) --- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 205b90e55..973ddc86a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -65,7 +65,7 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair) + const Derivation & drv, std::string_view name, RepairFlag repair) { auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) @@ -73,8 +73,8 @@ StorePath writeDerivation(ref store, /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(*store, false); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index c2df66229..7222d25e5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -79,7 +79,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath); From edc34cc1a2865777cee6b325f705fb218955b593 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:09:52 +0200 Subject: [PATCH 013/350] Add function for quoting strings (cherry picked from commit 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7) --- src/libstore/build.cc | 2 +- src/libstore/store-api.cc | 7 +------ src/libutil/util.cc | 22 ---------------------- src/libutil/util.hh | 22 ++++++++++++++++++++-- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9c6aedfa5..7efa81a62 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1904,7 +1904,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b043feb0a..aacbe5e0b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -741,12 +741,7 @@ std::string Store::showPaths(const StorePathSet & paths) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 012f1d071..52a1394f2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1205,28 +1205,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators) template vector tokenizeString(std::string_view s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3bfebcd15..559ed4332 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -345,8 +345,26 @@ template C tokenizeString(std::string_view s, const string & separators /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ From 1eb952d27ac0714a99a5f7b0e8c7034e7ac0bc0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 14:20:58 +0200 Subject: [PATCH 014/350] findAlongAttrPath(): Throw AttrPathNotFound (cherry picked from commit 6b0ca8e803710342af70e257935724c5ad84ca04) --- src/libexpr/attr-path.cc | 4 ++-- src/libexpr/attr-path.hh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 06b472d8b..843585631 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -70,7 +70,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; } @@ -82,7 +82,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, % attrPath % showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 716e5ba27..fcccc39c8 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,6 +7,8 @@ namespace nix { +MakeError(AttrPathNotFound, Error); + Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); From c1ca4f0accdb96295b27dadacd20b3db745e1d2d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Feb 2020 14:08:24 +0100 Subject: [PATCH 015/350] findAlongAttrPath(): Return position (cherry picked from commit 0b013a54dc570395bed887369f8dd622b8ce337b) --- src/libexpr/attr-path.cc | 9 ++++++--- src/libexpr/attr-path.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-env/nix-env.cc | 4 ++-- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/installables.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 843585631..1e22f5708 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -32,7 +32,7 @@ static Strings parseAttrPath(const string & s) } -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); @@ -41,6 +41,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Error(format("attribute selection path '%1%' does not match expression") % attrPath); Value * v = &vIn; + Pos pos = noPos; for (auto & attr : tokens) { @@ -72,6 +73,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, if (a == v->attrs->end()) throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; + pos = *a->pos; } else if (apType == apIndex) { @@ -85,11 +87,12 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; + pos = noPos; } } - return v; + return {v, pos}; } @@ -98,7 +101,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) Value * v2; try { auto dummyArgs = state.allocBindings(0); - v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; } catch (Error &) { throw Error("package '%s' has no source location information", what); } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index fcccc39c8..3e78e2899 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -9,7 +9,7 @@ namespace nix { MakeError(AttrPathNotFound, Error); -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 27ec7d0fe..1bda159d0 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -314,7 +314,7 @@ static void _main(int argc, char * * argv) state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); state->forceValue(v); getDerivations(*state, v, "", *autoArgs, drvs, false); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106dfe0b6..c7e553d0c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -178,7 +178,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first); getDerivations(state, v, pathPrefix, autoArgs, elems, true); @@ -408,7 +408,7 @@ static void queryInstSources(EvalState & state, Value vRoot; loadSourceExpr(state, instSource.nixExprPath, vRoot); for (auto & i : args) { - Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); } break; diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 5a886d69d..617d927a4 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -39,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v); PathSet context; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index cc0891811..18ced94b1 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -120,7 +120,7 @@ static int _main(int argc, char * * argv) Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); state->forceAttrs(v); /* Extract the URI. */ diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8ce6bd06e..eb6fca62b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -193,7 +193,7 @@ struct InstallableAttrPath : InstallableValue Bindings & autoArgs = *cmd.getAutoArgs(state); - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; state.forceValue(*v); return v; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 87f1f9d1b..c05c29517 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -145,7 +145,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto v = state->allocValue(); state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; return store->parseStorePath(state->forceString(*v2)); } From 231a8aa2c2196a3ba6d6bd3dc9d6c50e23e20a8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Feb 2020 14:22:01 +0100 Subject: [PATCH 016/350] nix edit: Support non-derivation attributes E.g. $ nix edit .#nixosConfigurations.bla now works. (cherry picked from commit d2032edb2f86e955a8a7724a27c0c3225f386500) --- src/libexpr/attr-path.cc | 2 +- src/libexpr/attr-path.hh | 1 + src/nix/command.hh | 9 ++------- src/nix/edit.cc | 10 ++++++++-- src/nix/eval.cc | 2 +- src/nix/installables.cc | 10 +++++----- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 1e22f5708..4545bfd72 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -103,7 +103,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto dummyArgs = state.allocBindings(0); v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; } catch (Error &) { - throw Error("package '%s' has no source location information", what); + throw NoPositionInfo("package '%s' has no source location information", what); } // FIXME: is it possible to extract the Pos object instead of doing this diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 3e78e2899..fce160da7 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -8,6 +8,7 @@ namespace nix { MakeError(AttrPathNotFound, Error); +MakeError(NoPositionInfo, Error); std::pair findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); diff --git a/src/nix/command.hh b/src/nix/command.hh index 00eb46903..a954a7d04 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -3,17 +3,12 @@ #include "args.hh" #include "common-eval-args.hh" #include "path.hh" +#include "eval.hh" namespace nix { extern std::string programPath; -struct Value; -class Bindings; -class EvalState; -struct Pos; -class Store; - /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -48,7 +43,7 @@ struct Installable Buildable toBuildable(); - virtual Value * toValue(EvalState & state) + virtual std::pair toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); } diff --git a/src/nix/edit.cc b/src/nix/edit.cc index ca410cd1f..1683eada0 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -29,9 +29,15 @@ struct CmdEdit : InstallableCommand { auto state = getEvalState(); - auto v = installable->toValue(*state); + auto [v, pos] = installable->toValue(*state); - Pos pos = findDerivationFilename(*state, *v, installable->what()); + try { + pos = findDerivationFilename(*state, *v, installable->what()); + } catch (NoPositionInfo &) { + } + + if (pos == noPos) + throw Error("cannot find position information for '%s", installable->what()); stopProgressBar(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 276fdf003..6398fc58e 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -52,7 +52,7 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state); + auto v = installable->toValue(*state).first; PathSet context; stopProgressBar(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index eb6fca62b..013218cd9 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -121,7 +121,7 @@ struct InstallableValue : Installable { auto state = cmd.getEvalState(); - auto v = toValue(*state); + auto v = toValue(*state).first; Bindings & autoArgs = *cmd.getAutoArgs(*state); @@ -169,11 +169,11 @@ struct InstallableExpr : InstallableValue std::string what() override { return text; } - Value * toValue(EvalState & state) override + std::pair toValue(EvalState & state) override { auto v = state.allocValue(); state.eval(state.parseExprFromString(text, absPath(".")), *v); - return v; + return {v, noPos}; } }; @@ -187,7 +187,7 @@ struct InstallableAttrPath : InstallableValue std::string what() override { return attrPath; } - Value * toValue(EvalState & state) override + std::pair toValue(EvalState & state) override { auto source = cmd.getSourceExpr(state); @@ -196,7 +196,7 @@ struct InstallableAttrPath : InstallableValue auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; state.forceValue(*v); - return v; + return {v, noPos}; } }; From 76e7d958ed8a1502cb060300093c120311d10cac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jan 2020 21:58:07 +0100 Subject: [PATCH 017/350] Fix coverage build https://hydra.nixos.org/build/110757285 (cherry picked from commit b430a81a1fbf6c792ba49e3aefe46256263430e5) --- src/libexpr/function-trace.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc index af1486f78..c6057b384 100644 --- a/src/libexpr/function-trace.cc +++ b/src/libexpr/function-trace.cc @@ -1,4 +1,5 @@ #include "function-trace.hh" +#include "logging.hh" namespace nix { From f9611c7ae48abd4506a93691c25f067a0e8e73b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:17:15 +0200 Subject: [PATCH 018/350] buildenv: Eliminate global variables, other cleanup (cherry picked from commit b82f75464d1e5ae9a00d8004e5dd7b1ca05059e4) --- src/libstore/build.cc | 1 + src/libstore/builtins.hh | 1 - src/libstore/builtins/buildenv.cc | 128 ++++++++++++++---------------- src/libstore/builtins/buildenv.hh | 21 +++++ 4 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 src/libstore/builtins/buildenv.hh diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 7efa81a62..dc6a2c606 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "builtins/buildenv.hh" #include "download.hh" #include "finally.hh" #include "compression.hh" diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 87d6ce665..66597e456 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 096593886..c1c85d0bf 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,4 @@ -#include "builtins.hh" +#include "buildenv.hh" #include #include @@ -7,16 +7,14 @@ namespace nix { -typedef std::map Priorities; - -// FIXME: change into local variables. - -static Priorities priorities; - -static unsigned long symlinks; +struct State +{ + std::map priorities; + unsigned long symlinks = 0; +}; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { DirEntries srcFiles; @@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); + createLinks(state, srcFile, dstFile, priority); continue; } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); @@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) throw SysError(format("unlinking '%1%'") % dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) @@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; + auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( "packages '%1%' and '%2%' have the same priority %3%; " @@ -109,67 +107,30 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) } createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + state.priorities[dstFile] = priority; + state.symlinks++; } } -typedef std::set FileProp; - -static FileProp done; -static FileProp postponed = FileProp{}; - -static Path out; - -static void addPkg(const Path & pkgDir, int priority) +void buildProfile(const Path & out, Packages && pkgs) { - if (!done.insert(pkgDir).second) return; - createLinks(pkgDir, out, priority); + State state; - try { - for (const auto & p : tokenizeString>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } -} + std::set done, postponed; -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} -}; + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); -typedef std::vector Packages; - -void builtinBuildenv(const BasicDerivation & drv) -{ - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; - - out = getAttr("out"); - createDirs(out); - - /* Convert the stuff we get from the environment back into a - * coherent data type. */ - Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; } - } + }; /* Symlink to the packages that have been installed explicitly by the * user. Process in priority order to reduce unnecessary @@ -189,13 +150,42 @@ void builtinBuildenv(const BasicDerivation & drv) */ auto priorityCounter = 1000; while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; + std::set pkgDirs; + postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", symlinks); + printError("created %d symlinks in user environment", state.symlinks); +} + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + buildProfile(out, std::move(pkgs)); createSymlink(getAttr("manifest"), out + "/manifest.nix"); } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 000000000..0a37459b0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} From 4260a22a55d7afc931b22e8c41282fbd0ed18a77 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jan 2020 15:34:06 +0100 Subject: [PATCH 019/350] absPath(): Use std::optional (cherry picked from commit 1bf9eb21b75f0d93d9c1633ea2e6fdf840047e79) --- src/libutil/util.cc | 6 +++--- src/libutil/util.hh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 52a1394f2..097ff210a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -97,10 +97,10 @@ void replaceEnv(std::map newEnv) } -Path absPath(Path path, Path dir) +Path absPath(Path path, std::optional dir) { if (path[0] != '/') { - if (dir == "") { + if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path lengths and doesn't define `PATH_MAX'. */ @@ -116,7 +116,7 @@ Path absPath(Path path, Path dir) free(buf); #endif } - path = dir + "/" + path; + path = *dir + "/" + path; } return canonPath(path); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 559ed4332..8c2cef9e1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -46,7 +46,7 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, Path dir = ""); +Path absPath(Path path, std::optional dir = {}); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink From 7a8de57d3e7e53a4f6d8060c7201f3fbbe6cd325 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Mar 2020 14:17:10 +0100 Subject: [PATCH 020/350] Pretty-print 'nix why-depends' / 'nix-store -q --tree' output Extracted from 678301072f05b650dc15c5edb4c25f08f0d6cace. --- src/libutil/util.hh | 34 ++++++++++++++++++++++++++++++++++ src/nix-store/nix-store.cc | 15 +++++---------- src/nix/why-depends.cc | 7 +------ tests/dependencies.sh | 8 ++++---- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8c2cef9e1..acef682cd 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -451,6 +451,13 @@ void ignoreException(); #define ANSI_BLUE "\e[34;1m" +/* Tree formatting. */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; + + /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, some escape sequences (such as colour setting) are copied but not @@ -576,4 +583,31 @@ extern PathFilter defaultPathFilter; AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +// A Rust/Python-like enumerate() iterator adapter. +// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +template ())), + typename = decltype(std::end(std::declval()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + bool operator != (const iterator & other) const { return iter != other.iter; } + void operator ++ () { ++i; ++iter; } + auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + auto begin() { return iterator{ 0, std::begin(iterable) }; } + auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward(iterable) }; +} + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c14d900ae..806ab7563 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -229,12 +229,6 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, /* Some code to print a tree representation of a derivation dependency graph. Topological sorting is used to keep the tree relatively flat. */ - -const string treeConn = "+---"; -const string treeLine = "| "; -const string treeNull = " "; - - static void printTree(const StorePath & path, const string & firstPad, const string & tailPad, StorePathSet & done) { @@ -254,10 +248,11 @@ static void printTree(const StorePath & path, auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); - for (auto i = sorted.begin(); i != sorted.end(); ++i) { - auto j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + for (const auto &[n, i] : enumerate(sorted)) { + bool last = n + 1 == sorted.size(); + printTree(i, + tailPad + (last ? treeLast : treeConn), + tailPad + (last ? treeNull : treeLine), done); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index c24ae7c8e..d3b7a674a 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -143,11 +143,6 @@ struct CmdWhyDepends : SourceExprCommand and `dependency`. */ std::function printNode; - const string treeConn = "╠═══"; - const string treeLast = "╚═══"; - const string treeLine = "║ "; - const string treeNull = " "; - struct BailOut { }; printNode = [&](Node & node, const string & firstPad, const string & tailPad) { @@ -157,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", firstPad, node.visited ? "\e[38;5;244m" : "", - firstPad != "" ? "=> " : "", + firstPad != "" ? "→ " : "", pathS); if (node.path == dependencyPath && !all diff --git a/tests/dependencies.sh b/tests/dependencies.sh index df204d185..8d0fdc10f 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix) echo "derivation is $drvPath" -nix-store -q --tree "$drvPath" | grep ' +---.*builder1.sh' +nix-store -q --tree "$drvPath" | grep '───.*builder1.sh' # Test Graphviz graph generation. nix-store -q --graph "$drvPath" > $TEST_ROOT/graph @@ -22,9 +22,9 @@ nix-store -q --graph "$outPath" > $TEST_ROOT/graph if test -n "$dot"; then # Does it parse? $dot < $TEST_ROOT/graph -fi +fi -nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2' +nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" @@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" +nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" From 777e21e596ec85f18eddf8b286d9ba4995010835 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Feb 2020 23:50:16 +0100 Subject: [PATCH 021/350] nix path-info --json: Print hash in SRI format (cherry picked from commit 442e665d6d3fcbdee7dece2f62a597142f8784b1) --- src/libstore/store-api.cc | 6 ++++-- src/libstore/store-api.hh | 1 + src/nix/path-info.cc | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index aacbe5e0b..b9e894a9a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -441,7 +441,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths, void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) + bool includeImpureInfo, bool showClosureSize, + Base hashBase, + AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); @@ -453,7 +455,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string()) + .attr("narHash", info->narHash.to_string(hashBase)) .attr("narSize", info->narSize); { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e0484de13..c57a6935e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -561,6 +561,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, + Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index bffa7b356..45ec297d2 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -89,7 +89,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON store->pathInfoToJSON(jsonRoot, // FIXME: preserve order? storePathsToSet(storePaths), - true, showClosureSize, AllowInvalid); + true, showClosureSize, SRI, AllowInvalid); } else { From 6b824c78f1f6c2de95b594a440118e259d28def0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Dec 2019 15:56:37 +0100 Subject: [PATCH 022/350] nix: Add --refresh as an alias for --tarball-ttl 0 (cherry picked from commit e721f99817bb7154d8098c902e25f84521a90b7f) --- src/nix/main.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 41d44f431..5cb1ad25e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -92,6 +92,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("no-net") .description("disable substituters and consider all previously downloaded files up-to-date") .handler([&]() { useNet = false; }); + + mkFlag() + .longName("refresh") + .description("consider all previously downloaded files out-of-date") + .handler([&]() { settings.tarballTtl = 0; }); } void printFlags(std::ostream & out) override From c85097da7c1ef06e2f1912c922110ff358de3aa2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Jan 2020 17:34:48 +0100 Subject: [PATCH 023/350] Fix --refresh with --no-net https://hydra.nixos.org/build/110879699 (cherry picked from commit 5bbe793abf18414878a069399d1759673d693fb6) --- src/nix/main.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5cb1ad25e..3b5f5516f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -55,6 +55,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; bool useNet = true; + bool refresh = false; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -96,7 +97,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("refresh") .description("consider all previously downloaded files out-of-date") - .handler([&]() { settings.tarballTtl = 0; }); + .handler([&]() { refresh = true; }); } void printFlags(std::ostream & out) override @@ -181,6 +182,9 @@ void mainWrapped(int argc, char * * argv) downloadSettings.connectTimeout = 1; } + if (args.refresh) + settings.tarballTtl = 0; + args.command->prepare(); args.command->run(); } From 0a10854f857e519d1455ff0f0be1f8401c40a11c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Mar 2020 14:26:13 +0100 Subject: [PATCH 024/350] Misc changes from the flakes branch --- .gitignore | 2 +- Makefile.config.in | 1 + src/libexpr/eval.hh | 2 +- src/libstore/build.cc | 16 ++++++++-------- src/libstore/builtins/buildenv.cc | 2 +- src/libstore/http-binary-cache-store.cc | 5 +++-- src/libstore/local-store.cc | 13 +++---------- src/libstore/nar-info-disk-cache.cc | 7 +------ src/libstore/nar-info-disk-cache.hh | 2 +- src/libstore/parsed-derivations.cc | 6 +++++- src/libstore/parsed-derivations.hh | 10 ++++++---- src/libstore/profiles.cc | 18 ++++++++++++++++++ src/libstore/profiles.hh | 4 ++++ src/libstore/sqlite.cc | 25 +++++++++++++++++++++++-- src/libstore/sqlite.hh | 10 +++++++--- src/libstore/store-api.hh | 5 +++++ src/libutil/hash.hh | 12 ++++++++++++ src/libutil/util.hh | 5 +++-- src/nix-env/nix-env.cc | 17 ++--------------- tests/binary-cache.sh | 4 ++-- tests/config.nix.in | 2 +- tests/fetchGit.sh | 2 ++ 22 files changed, 110 insertions(+), 60 deletions(-) diff --git a/.gitignore b/.gitignore index e10c75418..e3186fa76 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,7 @@ perl/Makefile.config /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/*.gen.hh +*.gen.* /src/nix/nix diff --git a/Makefile.config.in b/Makefile.config.in index fe609ce06..e7a12089a 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -37,6 +37,7 @@ prefix = @prefix@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ +system = @system@ doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5efad8707..1485dc7fe 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -368,7 +368,7 @@ struct EvalSettings : Config "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; Setting traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; }; extern EvalSettings evalSettings; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dc6a2c606..0e3a23a4d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1398,7 +1398,7 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); + lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { worker.waitForAWhile(shared_from_this()); @@ -1429,7 +1429,7 @@ void DerivationGoal::tryToBuild() for (auto & i : drv->outputs) { if (worker.store.isValidPath(i.second.path)) continue; debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); + deletePath(worker.store.Store::toRealPath(i.second.path)); } /* Don't do a remote build if the derivation has the attribute @@ -1686,7 +1686,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); + deletePath(worker.store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -2072,7 +2072,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2551,7 +2551,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; auto json = *structuredAttrs; @@ -2917,7 +2917,7 @@ void DerivationGoal::addDependency(const StorePath & path) #if __linux__ - Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); @@ -3579,7 +3579,7 @@ void DerivationGoal::registerOutputs() if (needsHashRewrite()) { auto r = redirectedOutputs.find(i.second.path); if (r != redirectedOutputs.end()) { - auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair && redirectedBadOutputs.count(i.second.path) && pathExists(redirected)) @@ -3672,7 +3672,7 @@ void DerivationGoal::registerOutputs() BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); + Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c1c85d0bf..1b802d908 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", state.symlinks); + debug("created %d symlinks in user environment", state.symlinks); } void builtinBuildenv(const BasicDerivation & drv) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index d4ae36662..011794c62 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -163,10 +163,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd2e86f29..ae7513ad8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,9 +298,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -312,11 +310,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ @@ -350,7 +343,7 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - const char * schema = + static const char schema[] = #include "schema.sql.gen.hh" ; db.exec(schema); @@ -1275,7 +1268,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) else hashSink = std::make_unique(info->narHash.type, storePathToHash(printStorePath(info->path))); - dumpPath(toRealPath(printStorePath(i)), *hashSink); + dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 907645d86..442541330 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 878acbb87..04de2c5eb 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,7 +10,7 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; - virtual ~NarInfoDiskCache() { }; + virtual ~NarInfoDiskCache() { } virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d0f289a0f..45c033c66 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,5 +1,7 @@ #include "parsed-derivations.hh" +#include + namespace nix { ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) @@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) { try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); + structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); } } } +ParsedDerivation::~ParsedDerivation() { } + std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index cec868754..f4df5dd54 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,6 +1,6 @@ #include "derivations.hh" -#include +#include namespace nix { @@ -8,15 +8,17 @@ class ParsedDerivation { StorePath drvPath; BasicDerivation & drv; - std::optional structuredAttrs; + std::unique_ptr structuredAttrs; public: ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); - const std::optional & getStructuredAttrs() const + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const { - return structuredAttrs; + return structuredAttrs.get(); } std::optional getStringAttr(const std::string & name) const; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index dae3f2d32..2bef51878 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 5fa1533de..78645d8b6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..eb1daafc5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index bd012d9b9..fd04c9b07 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -5,8 +5,8 @@ #include "types.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c57a6935e..0fa59be6a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -677,6 +677,11 @@ public: return storePath; } + Path toRealPath(const StorePath & storePath) + { + return toRealPath(printStorePath(storePath)); + } + virtual void createUser(const std::string & userName, uid_t userId) { } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ffa43ecf5..ea9fca3e7 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -80,6 +80,18 @@ struct Hash or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ std::string to_string(Base base = Base32, bool includeType = true) const; + + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } + + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } }; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index acef682cd..7c3a30242 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -476,10 +476,11 @@ string base64Decode(const string & s); /* Get a value for the specified key from an associate container, or a default value if the key doesn't exist. */ template -std::optional get(const T & map, const std::string & key) +std::optional get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - return i == map.end() ? std::optional() : i->second; + if (i == map.end()) return {}; + return std::optional(i->second); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c7e553d0c..e47b00acf 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1428,21 +1428,8 @@ static int _main(int argc, char * * argv) if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") { - Path profileLink = getHome() + "/.nix-profile"; - try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); - } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); - } catch (Error &) { - globals.profile = profileLink; - } - } + if (globals.profile == "") + globals.profile = getDefaultProfile(); op(globals, opFlags, opArgs); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..a3c3c7847 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -48,7 +48,7 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests @@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/config.nix.in b/tests/config.nix.in index 51aed539c..0ec2eba6b 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -3,7 +3,7 @@ rec { path = "@coreutils@"; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 4c46bdf04..ed8fa14d6 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,6 +9,8 @@ clearStore repo=$TEST_ROOT/git +export _NIX_FORCE_HTTP=1 + rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 git init $repo From 4171ab4bbd95e9bb4b9d6c6e8c143002d92b0c06 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 24 Mar 2020 09:18:23 -0600 Subject: [PATCH 025/350] renaming --- src/libutil/error.cc | 8 +++----- src/libutil/error.hh | 31 +++++++++++++++++++++++++++++-- tests/errors/main.cc | 9 +++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 71f422622..81c1f1805 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -41,8 +41,6 @@ void print_code_lines(string &prefix, NixCode &nix_code) % nix_code.errLine->errLineOfCode << endl; - - // error arrows for the column range. if (nix_code.errLine->columnRange.has_value()) { @@ -98,7 +96,7 @@ void print_error(ErrorInfo &einfo) } } - int ndl = level_string.length() + 3 + einfo.errName.length() + einfo.toolName.length(); + int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length(); int dashwidth = errwidth - 3 ? 3 : 80 - ndl; string dashes; @@ -110,9 +108,9 @@ void print_error(ErrorInfo &einfo) % prefix % level_string % "---" - % einfo.errName + % einfo.name % dashes - % einfo.toolName + % einfo.program << endl; // filename. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 8dac1508e..69776eb8c 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -39,13 +39,40 @@ class NixCode { class ErrorInfo { public: ErrLevel level; - string errName; + string name; string description; - string toolName; + string program; optional nixCode; string hint; + ErrorInfo& GetEI() { return *this; } }; +template +class AddName : private T +{ + public: + T& name(const std::string &name){ + GetEI().name = name; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +template +class AddDescription : private T +{ + public: + T& description(const std::string &description){ + GetEI().description = description; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +typedef AddName> StandardError; + string showErrLine(ErrLine &errLine); void print_code_lines(string &prefix, NixCode &nix_code); diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 1c998103e..0e6d781c9 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -24,13 +24,18 @@ using namespace nix; ErrorInfo generic; generic.level = elError; - generic.errName = "error name"; + generic.name = "error name"; generic.description = "general error description"; - generic.toolName = "nixtool.exe"; + generic.program = "nixtool.exe"; generic.nixCode = nixcode; print_error(generic); + + StandardError standardError; + + print_error(standardError.name("blah").description("blah")); + return 0; } From 0166e7ab6d1d4530e9d6d9562693a58ab54c36ba Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 24 Mar 2020 11:21:35 -0600 Subject: [PATCH 026/350] MkNixCode, MkErrLine approach --- src/libutil/error.cc | 6 +- src/libutil/error.hh | 187 +++++++++++++++++++++++++++++++++++++++---- tests/errors/main.cc | 38 ++++++++- 3 files changed, 211 insertions(+), 20 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 81c1f1805..f258d6a83 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -94,7 +94,11 @@ void print_error(ErrorInfo &einfo) level_string = "warning:"; // TODO make yellow. break; } - } + default: + { + level_string = "wat:"; + break; + }} int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length(); int dashwidth = errwidth - 3 ? 3 : 80 - ndl; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 69776eb8c..9449540af 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -3,6 +3,7 @@ #include "types.hh" #include #include +#include using std::string; using std::optional; @@ -16,35 +17,171 @@ enum ErrLevel class ColumnRange { public: - unsigned int start; - unsigned int len; + unsigned int start; + unsigned int len; }; +class ErrorInfo; + +// ------------------------------------------------- +// forward declarations before ErrLine. +template +class AddLineNumber; + +template +class AddColumnRange; + +template +class AddLOC; + class ErrLine { public: - int lineNumber; - optional columnRange; - optional prevLineOfCode; - string errLineOfCode; - optional nextLineOfCode; + int lineNumber; + optional columnRange; + optional prevLineOfCode; + string errLineOfCode; + optional nextLineOfCode; + + friend AddLineNumber; + friend AddColumnRange; + friend AddLOC; + ErrLine& GetEL() { return *this; } + private: + ErrLine() {} }; +template +class AddLineNumber : public T +{ + public: + T& lineNumber(int lineNumber) { + GetEL().lineNumber = lineNumber; + return *this; + } + protected: + ErrLine& GetEL() { return T::GetEL(); } +}; + +template +class AddColumnRange : public T +{ + public: + T& columnRange(unsigned int start, unsigned int len) { + GetEL().columnRange = { start, len }; + return *this; + } + protected: + ErrLine& GetEL() { return T::GetEL(); } +}; + +template +class AddLOC : public T +{ + public: + T& linesOfCode(optional prevloc, string loc, optional nextloc) { + GetEL().prevLineOfCode = prevloc; + GetEL().errLineOfCode = loc; + GetEL().nextLineOfCode = nextloc; + return *this; + } + protected: + ErrLine& GetEL() { return T::GetEL(); } +}; + +typedef AddLineNumber>> MkErrLine; +MkErrLine mkErrLine; + + +// ------------------------------------------------- +// NixCode. + +template +class AddNixFile; + +template +class AddErrLine; + class NixCode { public: - optional nixFile; - optional errLine; + optional nixFile; + optional errLine; + + friend AddNixFile; + friend AddErrLine; + friend ErrorInfo; + NixCode& GetNC() { return *this; } + private: + NixCode() {} }; +template +class AddNixFile : public T +{ + public: + T& nixFile(string filename) { + GetNC().nixFile = filename; + return *this; + } + protected: + NixCode& GetNC() { return T::GetNC(); } +}; + +template +class AddErrLine : public T +{ + public: + T& errLine(ErrLine errline) { + GetNC().errLine = errline; + return *this; + } + protected: + NixCode& GetNC() { return T::GetNC(); } +}; + +typedef AddNixFile> MkNixCode; + +// ------------------------------------------------- +// ErrorInfo. + +template +class AddName; + +template +class AddDescription; + +template +class AddNixCode; class ErrorInfo { public: - ErrLevel level; - string name; - string description; - string program; - optional nixCode; - string hint; - ErrorInfo& GetEI() { return *this; } + ErrLevel level; + string name; + string description; + string program; + optional nixCode; + string hint; + ErrorInfo& GetEI() { return *this; } + + // give these access to the private constructor, + // when they are direct descendants. + friend AddName; + friend AddDescription; + friend AddNixCode; + + protected: + ErrorInfo(ErrLevel level) { this->level = level; } +}; + +class EIError : public ErrorInfo +{ + protected: + EIError() : ErrorInfo(elError) {} +}; + +class EIWarning : public ErrorInfo +{ + protected: + EIWarning() : ErrorInfo(elWarning) {} }; template @@ -71,7 +208,23 @@ class AddDescription : private T ErrorInfo& GetEI() { return T::GetEI(); } }; -typedef AddName> StandardError; +template +class AddNixCode : private T +{ + public: + T& nixcode(const NixCode &nixcode){ + GetEI().nixCode = nixcode; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +typedef AddName> StandardError; +typedef AddName> StandardWarning; + +typedef AddName>> MkNixError; +typedef AddName>> MkNixWarning; string showErrLine(ErrLine &errLine); diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 0e6d781c9..20dba046b 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -4,11 +4,13 @@ #include using std::optional; +using std::nullopt; int main() { using namespace nix; + /* ColumnRange columnRange; columnRange.start = 24; columnRange.len = 14; @@ -30,11 +32,43 @@ using namespace nix; generic.nixCode = nixcode; print_error(generic); - + */ StandardError standardError; - print_error(standardError.name("blah").description("blah")); + print_error(standardError + .name("name") + .description("description")); + + StandardWarning standardWarning; + + print_error(standardWarning + .name("warning name") + .description("warning description")); + + print_error(MkNixError() + .name("name") + .description("description") + .nixcode( + MkNixCode() + .nixFile("myfile.nix") + .errLine(MkErrLine().lineNumber(40) + .columnRange(50,10) + .linesOfCode(nullopt + ,"this is the problem line of code" + ,nullopt)))); + + print_error(MkNixWarning() + .name("name") + .description("description") + .nixcode( + MkNixCode() + .nixFile("myfile.nix") + .errLine(MkErrLine().lineNumber(40) + .columnRange(50,10) + .linesOfCode(nullopt + ,"this is the problem line of code" + ,nullopt)))); return 0; } From 657c08c852ce94123984c475bc9d5f094f7d8d46 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 24 Mar 2020 13:26:20 -0600 Subject: [PATCH 027/350] fix column range --- tests/errors/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 20dba046b..c2c4d0060 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -53,7 +53,7 @@ using namespace nix; MkNixCode() .nixFile("myfile.nix") .errLine(MkErrLine().lineNumber(40) - .columnRange(50,10) + .columnRange(13,7) .linesOfCode(nullopt ,"this is the problem line of code" ,nullopt)))); @@ -65,7 +65,7 @@ using namespace nix; MkNixCode() .nixFile("myfile.nix") .errLine(MkErrLine().lineNumber(40) - .columnRange(50,10) + .columnRange(13,7) .linesOfCode(nullopt ,"this is the problem line of code" ,nullopt)))); From fc310eda3afdf97c55a1a5ac6756f8d54a2a5ad6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 24 Mar 2020 14:24:57 -0600 Subject: [PATCH 028/350] switch to one level of builder function, not subobject functions --- src/libutil/error.hh | 197 ++++++++++++++++++++----------------------- tests/errors/main.cc | 47 +++++------ 2 files changed, 112 insertions(+), 132 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 9449540af..25f3e8ee8 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -23,17 +23,6 @@ class ColumnRange { class ErrorInfo; -// ------------------------------------------------- -// forward declarations before ErrLine. -template -class AddLineNumber; - -template -class AddColumnRange; - -template -class AddLOC; - class ErrLine { public: int lineNumber; @@ -42,107 +31,25 @@ class ErrLine { string errLineOfCode; optional nextLineOfCode; - friend AddLineNumber; - friend AddColumnRange; - friend AddLOC; - ErrLine& GetEL() { return *this; } - private: - ErrLine() {} }; -template -class AddLineNumber : public T -{ - public: - T& lineNumber(int lineNumber) { - GetEL().lineNumber = lineNumber; - return *this; - } - protected: - ErrLine& GetEL() { return T::GetEL(); } -}; - -template -class AddColumnRange : public T -{ - public: - T& columnRange(unsigned int start, unsigned int len) { - GetEL().columnRange = { start, len }; - return *this; - } - protected: - ErrLine& GetEL() { return T::GetEL(); } -}; - -template -class AddLOC : public T -{ - public: - T& linesOfCode(optional prevloc, string loc, optional nextloc) { - GetEL().prevLineOfCode = prevloc; - GetEL().errLineOfCode = loc; - GetEL().nextLineOfCode = nextloc; - return *this; - } - protected: - ErrLine& GetEL() { return T::GetEL(); } -}; - -typedef AddLineNumber>> MkErrLine; -MkErrLine mkErrLine; - - -// ------------------------------------------------- -// NixCode. - -template -class AddNixFile; - -template -class AddErrLine; - class NixCode { public: optional nixFile; optional errLine; - friend AddNixFile; - friend AddErrLine; - friend ErrorInfo; - NixCode& GetNC() { return *this; } - private: - NixCode() {} -}; - -template -class AddNixFile : public T -{ - public: - T& nixFile(string filename) { - GetNC().nixFile = filename; - return *this; + ErrLine& ensureErrLine() + { + if (!this->errLine.has_value()) + this->errLine = optional(ErrLine()); + return *this->errLine; } - protected: - NixCode& GetNC() { return T::GetNC(); } }; -template -class AddErrLine : public T -{ - public: - T& errLine(ErrLine errline) { - GetNC().errLine = errline; - return *this; - } - protected: - NixCode& GetNC() { return T::GetNC(); } -}; - -typedef AddNixFile> MkNixCode; - // ------------------------------------------------- // ErrorInfo. +// Forward friend class declarations. "builder classes" template class AddName; @@ -152,6 +59,22 @@ class AddDescription; template class AddNixCode; +template +class AddNixFile; + +template +class AddErrLine; + +template +class AddLineNumber; + +template +class AddColumnRange; + +template +class AddLOC; + +// The error info class itself. class ErrorInfo { public: ErrLevel level; @@ -167,9 +90,23 @@ class ErrorInfo { friend AddName; friend AddDescription; friend AddNixCode; + friend AddNixFile; + friend AddErrLine; + friend AddLineNumber; + friend AddColumnRange; + friend AddLOC; + NixCode& ensureNixCode() + { + if (!this->nixCode.has_value()) + this->nixCode = optional(NixCode()); + return *this->nixCode; + } protected: + // constructor is protected, so only the builder classes can create an ErrorInfo. ErrorInfo(ErrLevel level) { this->level = level; } + + }; class EIError : public ErrorInfo @@ -208,23 +145,73 @@ class AddDescription : private T ErrorInfo& GetEI() { return T::GetEI(); } }; -template -class AddNixCode : private T +template +class AddNixFile : public T { public: - T& nixcode(const NixCode &nixcode){ - GetEI().nixCode = nixcode; + T& nixFile(string filename) { + GetEI().ensureNixCode().nixFile = filename; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +template +class AddLineNumber : public T +{ + public: + T& lineNumber(int lineNumber) { + GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; return *this; } protected: ErrorInfo& GetEI() { return T::GetEI(); } }; +template +class AddColumnRange : public T +{ + public: + T& columnRange(unsigned int start, unsigned int len) { + GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +template +class AddLOC : public T +{ + public: + T& linesOfCode(optional prevloc, string loc, optional nextloc) { + GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; + GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; + GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +typedef AddNixFile> MkNixCode; + typedef AddName> StandardError; typedef AddName> StandardWarning; -typedef AddName>> MkNixError; -typedef AddName>> MkNixWarning; +typedef AddName< + AddDescription< + AddNixFile< + AddLineNumber< + AddColumnRange< + AddLOC>>>>> MkNixError; +typedef AddName< + AddDescription< + AddNixFile< + AddLineNumber< + AddColumnRange< + AddLOC>>>>> MkNixWarning; string showErrLine(ErrLine &errLine); diff --git a/tests/errors/main.cc b/tests/errors/main.cc index c2c4d0060..6b024d287 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -34,41 +34,34 @@ using namespace nix; print_error(generic); */ - StandardError standardError; - - print_error(standardError + print_error(StandardError() .name("name") .description("description")); - StandardWarning standardWarning; - - print_error(standardWarning + print_error(StandardWarning() .name("warning name") .description("warning description")); - print_error(MkNixError() - .name("name") - .description("description") - .nixcode( - MkNixCode() - .nixFile("myfile.nix") - .errLine(MkErrLine().lineNumber(40) - .columnRange(13,7) - .linesOfCode(nullopt - ,"this is the problem line of code" - ,nullopt)))); print_error(MkNixWarning() - .name("name") - .description("description") - .nixcode( - MkNixCode() - .nixFile("myfile.nix") - .errLine(MkErrLine().lineNumber(40) - .columnRange(13,7) - .linesOfCode(nullopt - ,"this is the problem line of code" - ,nullopt)))); + .name("warning name") + .description("warning description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(nullopt + ,"this is the problem line of code" + ,nullopt)); + + print_error(MkNixError() + .name("error name") + .description("warning description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(nullopt + ,"this is the problem line of code" + ,nullopt)); return 0; } From 7313aa267b5be1e5264e4577e7bc3daec2fef282 Mon Sep 17 00:00:00 2001 From: Greg Price Date: Tue, 24 Mar 2020 21:15:01 -0700 Subject: [PATCH 029/350] installer: Fix terminal colors. The install-multi-user script uses blue, green, and red colors, as well as bold and underline, to add helpful formatting that helps structure its rather voluminous output. Unfortunately, the terminal escape sequences it uses are not quite well-formed. The relevant information is all there, just obscured by some extra noise, a leading parameter `38`. Empirically, the result is: * On macOS, in both Terminal.app and iTerm2, the spurious `38` is ignored, the rest of the escape sequence is applied, and the colors show up as intended. * On Linux, in at least gnome-terminal and xterm, the spurious `38` and the next parameter after it are ignored, and what's left is applied. So in the sequence `38;4;32`, the 4 (underline) is ignored but the 32 (green) takes effect; in a more typical sequence like `38;34`, the 34 (blue) is ignored and nothing happens. These codes are all unchanged since this script's origins as a Darwin-only script -- so the fact that they work fine in common macOS terminals goes some way to explain how the bug arose. Happily, we can make the colors work as intended by just deleting the extra `38;`. Tested in all four terminals mentioned above; the new codes work correctly on all of them, and on the two macOS terminals they work exactly the same as before. --- In a bit more technical detail -- perhaps more than anyone, me included, ever wanted to know, but now that I've gone and learned it I'll write it down anyway :) -- here's what's happening in these codes: An ECMA-48 "control sequence" begins with `\033[` aka "CSI", contains any number of parameters as semicolon-separated decimal numbers (plus sometimes other wrinkles), and ends with a byte from 0x40..0x7e. In our case, with `m` aka "SGR", "Select Graphic Rendition". An SGR control sequence `\033[...m` sets colors, fonts, text styles, etc. In particular a parameter `31` means red, `32` green, `34` blue, `4` underline, and `0` means reset to normal. Those are all we use. There is also a `38`. This is used for setting colors too... but it needs arguments. `38;5;nn` is color nn from a 256-color palette, and `38;2;rr;gg;bb` has the given RGB values. There is no meaning defined for `38;1` or `38;34` etc. On seeing a parameter `38` followed by an unrecognized argument for it, apparently some implementations (as seen on macOS) discard only the `38` and others (as seen on Linux) discard the argument too before resuming. --- scripts/install-multi-user.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 35341543e..a0f1deb98 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -13,12 +13,12 @@ set -o pipefail # however tracking which bits came from which would be impossible. readonly ESC='\033[0m' -readonly BOLD='\033[38;1m' -readonly BLUE='\033[38;34m' -readonly BLUE_UL='\033[38;4;34m' -readonly GREEN='\033[38;32m' -readonly GREEN_UL='\033[38;4;32m' -readonly RED='\033[38;31m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" From 3582dc3c88d38d5c46091b4a23601d6f56a4c81e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 25 Mar 2020 10:52:03 -0600 Subject: [PATCH 030/350] programName as static member var --- src/libutil/error.cc | 13 +++++++++---- src/libutil/error.hh | 6 +++++- tests/errors/main.cc | 4 ++++ 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index f258d6a83..4abee052d 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,11 +1,15 @@ #include "error.hh" #include +#include namespace nix { using std::cout; using std::endl; +using std::nullopt; + +optional ErrorInfo::programName = nullopt; // return basic_format? string showErrLine(ErrLine &errLine) @@ -98,10 +102,11 @@ void print_error(ErrorInfo &einfo) { level_string = "wat:"; break; - }} + } + } - int ndl = level_string.length() + 3 + einfo.name.length() + einfo.program.length(); - int dashwidth = errwidth - 3 ? 3 : 80 - ndl; + int ndl = prefix.length() + level_string.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; string dashes; for (int i = 0; i < dashwidth; ++i) @@ -114,7 +119,7 @@ void print_error(ErrorInfo &einfo) % "---" % einfo.name % dashes - % einfo.program + % einfo.programName.value_or("") << endl; // filename. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 25f3e8ee8..7260dfd5b 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -80,11 +80,12 @@ class ErrorInfo { ErrLevel level; string name; string description; - string program; optional nixCode; string hint; ErrorInfo& GetEI() { return *this; } + static optional programName; + // give these access to the private constructor, // when they are direct descendants. friend AddName; @@ -109,18 +110,21 @@ class ErrorInfo { }; +// Init as error class EIError : public ErrorInfo { protected: EIError() : ErrorInfo(elError) {} }; +// Init as warning class EIWarning : public ErrorInfo { protected: EIWarning() : ErrorInfo(elWarning) {} }; +// Builder class definitions. template class AddName : private T { diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 6b024d287..5ac429b46 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -5,11 +5,15 @@ using std::optional; using std::nullopt; +using std::cout; +using std::endl; int main() { using namespace nix; + ErrorInfo::programName = optional("errorTest"); + /* ColumnRange columnRange; columnRange.start = 24; From d44c9c55817839a41a7b4c509da027d7b4176ca5 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 25 Mar 2020 11:20:44 -0600 Subject: [PATCH 031/350] some colors --- src/libutil/error.cc | 15 ++++++++++----- tests/errors/main.cc | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4abee052d..9fbd234e0 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -62,7 +62,7 @@ void print_code_lines(string &prefix, NixCode &nix_code) arrows.append("^"); } - cout << format("%1% |%2%%3%") % prefix % spaces % arrows; + cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows; } @@ -90,12 +90,16 @@ void print_error(ErrorInfo &einfo) { case ErrLevel::elError: { - level_string = "error:"; // TODO make red. + level_string = ANSI_RED; + level_string += "error:"; + level_string += ANSI_NORMAL; break; } case ErrLevel::elWarning: { - level_string = "warning:"; // TODO make yellow. + level_string = ANSI_YELLOW; + level_string += "warning:"; + level_string += ANSI_NORMAL; break; } default: @@ -113,7 +117,7 @@ void print_error(ErrorInfo &einfo) dashes.append("-"); // divider. - cout << format("%1%%2% %3% %4% %5% %6%") + cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) % prefix % level_string % "---" @@ -131,7 +135,8 @@ void print_error(ErrorInfo &einfo) ? string(" ") + showErrLine(*einfo.nixCode->errLine) : ""; - cout << format("%1%in file: %2%%3%") % prefix % *einfo.nixCode->nixFile % eline << endl; + cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) + % prefix % *einfo.nixCode->nixFile % eline << endl; cout << prefix << endl; } else diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 5ac429b46..f53101629 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -59,7 +59,7 @@ using namespace nix; print_error(MkNixError() .name("error name") - .description("warning description") + .description("error description") .nixFile("myfile.nix") .lineNumber(40) .columnRange(13,7) From a3ef00be6c668bd59a879a4acdb1599225d86b44 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 27 Mar 2020 10:03:02 -0600 Subject: [PATCH 032/350] camelcase; optional hint --- src/libutil/error.cc | 60 +++++++++++++++++--------------- src/libutil/error.hh | 82 +++++++++++++++++++++++++++++++++++++------- tests/errors/main.cc | 34 +++++++++--------- 3 files changed, 119 insertions(+), 57 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 9fbd234e0..d6595070f 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -24,55 +24,55 @@ string showErrLine(ErrLine &errLine) }; } -void print_code_lines(string &prefix, NixCode &nix_code) +void printCodeLines(string &prefix, NixCode &nixCode) { - if (nix_code.errLine.has_value()) + if (nixCode.errLine.has_value()) { // previous line of code. - if (nix_code.errLine->prevLineOfCode.has_value()) { + if (nixCode.errLine->prevLineOfCode.has_value()) { cout << format("%1% %|2$5d|| %3%") % prefix - % (nix_code.errLine->lineNumber - 1) - % *nix_code.errLine->prevLineOfCode + % (nixCode.errLine->lineNumber - 1) + % *nixCode.errLine->prevLineOfCode << endl; } // line of code containing the error.%2$+5d% cout << format("%1% %|2$5d|| %3%") % prefix - % (nix_code.errLine->lineNumber) - % nix_code.errLine->errLineOfCode + % (nixCode.errLine->lineNumber) + % nixCode.errLine->errLineOfCode << endl; // error arrows for the column range. - if (nix_code.errLine->columnRange.has_value()) + if (nixCode.errLine->columnRange.has_value()) { - int start = nix_code.errLine->columnRange->start; + int start = nixCode.errLine->columnRange->start; std::string spaces; for (int i = 0; i < start; ++i) { spaces.append(" "); } - int len = nix_code.errLine->columnRange->len; + int len = nixCode.errLine->columnRange->len; std::string arrows; for (int i = 0; i < len; ++i) { arrows.append("^"); } - cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows; + cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl; } // next line of code. - if (nix_code.errLine->nextLineOfCode.has_value()) { + if (nixCode.errLine->nextLineOfCode.has_value()) { cout << format("%1% %|2$5d|| %3%") % prefix - % (nix_code.errLine->lineNumber + 1) - % *nix_code.errLine->nextLineOfCode + % (nixCode.errLine->lineNumber + 1) + % *nixCode.errLine->nextLineOfCode << endl; } @@ -80,36 +80,36 @@ void print_code_lines(string &prefix, NixCode &nix_code) } -void print_error(ErrorInfo &einfo) +void printErrorInfo(ErrorInfo &einfo) { int errwidth = 80; string prefix = " "; - string level_string; + string levelString; switch (einfo.level) { case ErrLevel::elError: { - level_string = ANSI_RED; - level_string += "error:"; - level_string += ANSI_NORMAL; + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; break; } case ErrLevel::elWarning: { - level_string = ANSI_YELLOW; - level_string += "warning:"; - level_string += ANSI_NORMAL; + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; break; } default: { - level_string = "wat:"; + levelString = "wat:"; break; } } - int ndl = prefix.length() + level_string.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; string dashes; @@ -119,7 +119,7 @@ void print_error(ErrorInfo &einfo) // divider. cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) % prefix - % level_string + % levelString % "---" % einfo.name % dashes @@ -152,11 +152,15 @@ void print_error(ErrorInfo &einfo) // lines of code. if (einfo.nixCode.has_value()) - print_code_lines(prefix, *einfo.nixCode); + printCodeLines(prefix, *einfo.nixCode); // hint - cout << prefix << einfo.hint << endl; - cout << prefix << endl; + if (einfo.hint.has_value()) + { + cout << prefix << endl; + cout << prefix << *einfo.hint << endl; + cout << prefix << endl; + } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7260dfd5b..f02d7288a 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,12 +1,16 @@ #pragma once -#include "types.hh" +#include "util.hh" #include #include #include +#include using std::string; using std::optional; +using boost::format; +using std::cout; +using std::endl; namespace nix { @@ -81,13 +85,13 @@ class ErrorInfo { string name; string description; optional nixCode; - string hint; + optional hint; ErrorInfo& GetEI() { return *this; } static optional programName; // give these access to the private constructor, - // when they are direct descendants. + // when they are direct descendants (children but not grandchildren). friend AddName; friend AddDescription; friend AddNixCode; @@ -150,7 +154,7 @@ class AddDescription : private T }; template -class AddNixFile : public T +class AddNixFile : private T { public: T& nixFile(string filename) { @@ -162,7 +166,7 @@ class AddNixFile : public T }; template -class AddLineNumber : public T +class AddLineNumber : private T { public: T& lineNumber(int lineNumber) { @@ -174,7 +178,7 @@ class AddLineNumber : public T }; template -class AddColumnRange : public T +class AddColumnRange : private T { public: T& columnRange(unsigned int start, unsigned int len) { @@ -186,7 +190,7 @@ class AddColumnRange : public T }; template -class AddLOC : public T +class AddLOC : private T { public: T& linesOfCode(optional prevloc, string loc, optional nextloc) { @@ -199,17 +203,64 @@ class AddLOC : public T ErrorInfo& GetEI() { return T::GetEI(); } }; -typedef AddNixFile> MkNixCode; +template +class yellowify +{ +public: + yellowify(T &s) : value(s) {} + T &value; +}; -typedef AddName> StandardError; -typedef AddName> StandardWarning; +template +std::ostream& operator<<(std::ostream &out, const yellowify &y) +{ + return out << ANSI_YELLOW << y.value << ANSI_NORMAL; +} + +// hint format shows templated values in yellow. +class hintfmt +{ + public: + hintfmt(string format) :fmt(format) {} + template + hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; } + + template + friend class AddHint; + private: + format fmt; + +}; + +template +class AddHint : private T +{ + public: + T& hint(hintfmt &hfmt) { + GetEI().hint = optional(hfmt.fmt.str()); + return *this; + } + T& nohint() { + GetEI().hint = std::nullopt; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } +}; + +// -------------------------------------------------------- +// error types + +typedef AddName>> StandardError; +typedef AddName>> StandardWarning; typedef AddName< AddDescription< AddNixFile< AddLineNumber< AddColumnRange< - AddLOC>>>>> MkNixError; + AddLOC< + AddHint>>>>>> MkNixError; typedef AddName< AddDescription< AddNixFile< @@ -217,10 +268,15 @@ typedef AddName< AddColumnRange< AddLOC>>>>> MkNixWarning; + +// -------------------------------------------------------- +// error printing + +void printErrorInfo(ErrorInfo &einfo); + string showErrLine(ErrLine &errLine); -void print_code_lines(string &prefix, NixCode &nix_code); +void printCodeLines(string &prefix, NixCode &nixCode); -void print_error(ErrorInfo &einfo); } diff --git a/tests/errors/main.cc b/tests/errors/main.cc index f53101629..35d2a1cdf 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -8,9 +8,9 @@ using std::nullopt; using std::cout; using std::endl; -int main() { - -using namespace nix; +int main() +{ + using namespace nix; ErrorInfo::programName = optional("errorTest"); @@ -35,19 +35,23 @@ using namespace nix; generic.program = "nixtool.exe"; generic.nixCode = nixcode; - print_error(generic); + printErrorInfo(generic); */ - print_error(StandardError() + printErrorInfo(StandardError() .name("name") - .description("description")); + .description("description") + .nohint() + ); - print_error(StandardWarning() + printErrorInfo(StandardWarning() .name("warning name") - .description("warning description")); + .description("warning description") + .nohint() + ); - print_error(MkNixWarning() + printErrorInfo(MkNixWarning() .name("warning name") .description("warning description") .nixFile("myfile.nix") @@ -57,19 +61,17 @@ using namespace nix; ,"this is the problem line of code" ,nullopt)); - print_error(MkNixError() + printErrorInfo(MkNixError() .name("error name") .description("error description") .nixFile("myfile.nix") .lineNumber(40) .columnRange(13,7) - .linesOfCode(nullopt + .linesOfCode(optional("previous line of code") ,"this is the problem line of code" - ,nullopt)); + ,optional("next line of code")) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); return 0; } - - - - From 00eb3fcb7a19a38d5ca94f4dcbda149f16e73a6e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 27 Mar 2020 10:13:46 -0600 Subject: [PATCH 033/350] more cleanup --- src/libutil/error.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index d6595070f..9194539a5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -104,13 +104,13 @@ void printErrorInfo(ErrorInfo &einfo) } default: { - levelString = "wat:"; + levelString = format("invalid error level: %1%") % einfo.level; break; } } int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); - int dashwidth = ndl > (errwidth - 3) ? 3 : 80 - ndl; + int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; string dashes; for (int i = 0; i < dashwidth; ++i) @@ -161,7 +161,6 @@ void printErrorInfo(ErrorInfo &einfo) cout << prefix << *einfo.hint << endl; cout << prefix << endl; } - } } From 759f39800bb80eaa297ea2532b1640959487ed9c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 27 Mar 2020 10:55:09 -0600 Subject: [PATCH 034/350] remove util.hh from deps --- src/libutil/ansicolor.hh | 13 +++++++++++++ src/libutil/error.cc | 2 +- src/libutil/error.hh | 25 ++++++++++++++++--------- src/libutil/util.hh | 10 +--------- 4 files changed, 31 insertions(+), 19 deletions(-) create mode 100644 src/libutil/ansicolor.hh diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh new file mode 100644 index 000000000..390bd4d17 --- /dev/null +++ b/src/libutil/ansicolor.hh @@ -0,0 +1,13 @@ +#pragma once + +namespace nix +{ + /* Some ANSI escape sequences. */ + #define ANSI_NORMAL "\e[0m" + #define ANSI_BOLD "\e[1m" + #define ANSI_FAINT "\e[2m" + #define ANSI_RED "\e[31;1m" + #define ANSI_GREEN "\e[32;1m" + #define ANSI_YELLOW "\e[33;1m" + #define ANSI_BLUE "\e[34;1m" +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 9194539a5..c986affcc 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -104,7 +104,7 @@ void printErrorInfo(ErrorInfo &einfo) } default: { - levelString = format("invalid error level: %1%") % einfo.level; + levelString = (format("invalid error level: %1%") % einfo.level).str(); break; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f02d7288a..88980afb7 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,11 +1,13 @@ #pragma once -#include "util.hh" +#include "ansicolor.hh" #include #include #include #include +#include + using std::string; using std::optional; using boost::format; @@ -251,8 +253,14 @@ class AddHint : private T // -------------------------------------------------------- // error types -typedef AddName>> StandardError; -typedef AddName>> StandardWarning; +typedef AddName< + AddDescription< + AddHint< + EIError>>> StandardError; +typedef AddName< + AddDescription< + AddHint< + EIWarning>>> StandardWarning; typedef AddName< AddDescription< @@ -260,13 +268,16 @@ typedef AddName< AddLineNumber< AddColumnRange< AddLOC< - AddHint>>>>>> MkNixError; + AddHint< + EIError>>>>>>> MkNixError; typedef AddName< AddDescription< AddNixFile< AddLineNumber< AddColumnRange< - AddLOC>>>>> MkNixWarning; + AddLOC< + AddHint< + EIWarning>>>>>>> MkNixWarning; // -------------------------------------------------------- @@ -274,9 +285,5 @@ typedef AddName< void printErrorInfo(ErrorInfo &einfo); -string showErrLine(ErrLine &errLine); - -void printCodeLines(string &prefix, NixCode &nixCode); - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3bfebcd15..d69e29158 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "logging.hh" +#include "ansicolor.hh" #include #include @@ -423,15 +424,6 @@ std::string shellEscape(const std::string & s); void ignoreException(); -/* Some ANSI escape sequences. */ -#define ANSI_NORMAL "\e[0m" -#define ANSI_BOLD "\e[1m" -#define ANSI_FAINT "\e[2m" -#define ANSI_RED "\e[31;1m" -#define ANSI_GREEN "\e[32;1m" -#define ANSI_YELLOW "\e[33;1m" -#define ANSI_BLUE "\e[34;1m" - /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, From 225e62a56a7cebb030bebffb8d2bd7afe21cc64a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 29 Mar 2020 01:04:55 -0400 Subject: [PATCH 035/350] Replace some `bool recursive` with a new `FileIngestionMethod` enum --- perl/lib/Nix/Store.xs | 4 ++-- src/libexpr/eval.cc | 2 +- src/libexpr/primops.cc | 16 ++++++++-------- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build.cc | 12 ++++++------ src/libstore/daemon.cc | 24 ++++++++++++++++-------- src/libstore/derivations.cc | 6 +++--- src/libstore/derivations.hh | 2 +- src/libstore/download.cc | 9 +++++---- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 12 ++++++------ src/libstore/local-store.hh | 4 ++-- src/libstore/path.hh | 5 +++++ src/libstore/remote-store.cc | 12 +++++++----- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 18 +++++++++--------- src/libstore/store-api.hh | 11 +++++------ src/nix-prefetch-url/nix-prefetch-url.cc | 9 ++++++--- src/nix-store/nix-store.cc | 8 ++++---- src/nix/add-to-store.cc | 4 ++-- src/nix/command.cc | 10 +++++----- src/nix/command.hh | 4 ++-- src/nix/copy.cc | 2 +- src/nix/make-content-addressable.cc | 4 ++-- 27 files changed, 105 insertions(+), 87 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 1ca734e75..eab8ccacb 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -274,7 +274,7 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo)); + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, (FileIngestionMethod) recursive, parseHashType(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -285,7 +285,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { Hash h(hash, parseHashType(algo)); - auto path = store()->makeFixedOutputPath(recursive, h, name); + auto path = store()->makeFixedOutputPath((FileIngestionMethod) recursive, h, name); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..88bbf3b32 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1661,7 +1661,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) else { auto p = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..7d45733f4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional outputHash; std::string outputHashAlgo; - bool outputHashRecursive = false; + FileIngestionMethod outputHashRecursive = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); @@ -574,8 +574,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; + if (s == "recursive") outputHashRecursive = FileIngestionMethod::Recursive; + else if (s == "flat") outputHashRecursive = FileIngestionMethod::Flat; else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); }; @@ -725,7 +725,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), - (outputHashRecursive ? "r:" : "") + printHashType(h.type), + (static_cast(outputHashRecursive) ? "r:" : "") + printHashType(h.type), h.to_string(Base16, false))); } @@ -1038,7 +1038,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1095,7 +1095,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args if (args[0]->type != tLambda) throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1104,7 +1104,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Path path; string name; Value * filterFun = nullptr; - auto recursive = true; + auto recursive = FileIngestionMethod::Recursive; Hash expectedHash; for (auto & attr : *args[0]->attrs) { @@ -1120,7 +1120,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value state.forceValue(*attr.value); filterFun = attr.value; } else if (n == "recursive") - recursive = state.forceBool(*attr.value, *attr.pos); + recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..9f2b673ad 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -69,7 +69,7 @@ GitInfo exportGit(ref store, const std::string & uri, return files.count(file); }; - gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); + gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, FileIngestionMethod::Recursive, htSHA256, filter)); return gitInfo; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..548f4e392 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -63,7 +63,7 @@ HgInfo exportMercurial(ref store, const std::string & uri, return files.count(file); }; - hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); + hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, FileIngestionMethod::Recursive, htSHA256, filter)); return hgInfo; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a2d84861..a61af4a00 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive) { + if (recursive == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index aa13c1cb4..1a1ea636c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -79,7 +79,7 @@ public: std::shared_ptr accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e3a23a4d..224633106 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2712,7 +2712,7 @@ struct RestrictedStore : public LocalFSStore { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2725,7 +2725,7 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); goal.addDependency(path); @@ -3647,10 +3647,10 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - bool recursive; Hash h; + FileIngestionMethod recursive; Hash h; i.second.parseHashInfo(recursive, h); - if (!recursive) { + if (!static_cast(recursive)) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3659,7 +3659,7 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = static_cast(recursive) ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); @@ -3912,7 +3912,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); + auto used = static_cast(recursive) ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8e9f9d71b..f1afdff69 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -355,20 +355,24 @@ static void performOp(TunnelLogger * logger, ref store, } case wopAddToStore: { - bool fixed, recursive; std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; + FileIngestionMethod method; + { + bool fixed, recursive; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + method = FileIngestionMethod::Recursive; + } } HashType hashAlgo = parseHashType(s); TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ @@ -380,7 +384,11 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump( + method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + baseName, + method, + hashAlgo); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ddc86a..5934c1912 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,13 +9,13 @@ namespace nix { -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const { - recursive = false; + recursive = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = true; + recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 7222d25e5..b1224b93b 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -22,7 +22,7 @@ struct DerivationOutput , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } - void parseHashInfo(bool & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; typedef std::map DerivationOutputs; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 149c84765..8a9b65899 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -814,7 +814,8 @@ CachedDownloadResult Downloader::downloadCached( std::optional expectedStorePath; if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + auto method = request.unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + expectedStorePath = store->makeFixedOutputPath(method, request.expectedHash, name); if (store->isValidPath(*expectedStorePath)) { CachedDownloadResult result; result.storePath = store->printStorePath(*expectedStorePath); @@ -875,10 +876,10 @@ CachedDownloadResult Downloader::downloadCached( StringSink sink; dumpString(*res.data, sink); Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); store->addToStore(info, sink.s, NoRepair, NoCheckSigs); storePath = info.path.clone(); } @@ -914,7 +915,7 @@ CachedDownloadResult Downloader::downloadCached( if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); } replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink); storePath = std::move(*unpackedStorePath); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 458266be0..af20d389b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -195,7 +195,7 @@ struct LegacySSHStore : public Store { unsupported("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ae7513ad8..746f81beb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -557,7 +557,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - bool recursive; Hash h; + FileIngestionMethod recursive; Hash h; out->second.parseHashInfo(recursive, h); check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); @@ -1043,7 +1043,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); @@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive) { + if (recursive == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive) { + if (recursive == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1111,7 +1111,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (recursive == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref(readFile(srcPath)); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 16aeab0ad..c1e75390c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -149,7 +149,7 @@ public: std::shared_ptr accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained @@ -157,7 +157,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index c90bb1fff..73c4b8e29 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -73,6 +73,11 @@ const size_t storePathHashLen = 32; // i.e. 160 bits /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; +enum struct FileIngestionMethod : bool { + Flat = false, + Recursive = true +}; + struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..5c36693e6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -484,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -492,10 +492,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, Path srcPath(absPath(_srcPath)); - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); + conn->to + << wopAddToStore + << name + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashAlgo); try { conn->to.written = 0; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f301a97d8..3c86b4524 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,7 @@ public: std::shared_ptr accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b9e894a9a..06aa0883c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -171,18 +171,18 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - bool recursive, + FileIngestionMethod recursive, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive) { + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + + "fixed:out:" + (static_cast(recursive) ? (string) "r:" : "") + hash.to_string(Base16) + ":"), name); } } @@ -200,9 +200,9 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); + Hash h = static_cast(recursive) ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); return std::make_pair(makeFixedOutputPath(recursive, h, name), h); } @@ -781,8 +781,8 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); + FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; + Hash hash(std::string(ca, static_cast(recursive) ? 8 : 6)); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { @@ -826,9 +826,9 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(bool recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); + return "fixed:" + (static_cast(recursive) ? (std::string) "r:" : "") + hash.to_string(); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0fa59be6a..32c24c500 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -43,7 +43,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; @@ -346,7 +345,7 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(bool recursive, + StorePath makeFixedOutputPath(FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references = {}, bool hasSelfReference = false) const; @@ -358,7 +357,7 @@ public: store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive = true, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -462,12 +461,12 @@ public: The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -850,7 +849,7 @@ std::optional decodeValidPathInfo( /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 18ced94b1..8c364a86a 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -159,7 +159,8 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); - storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(recursive, expectedHash, name); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -208,13 +209,15 @@ static int _main(int argc, char * * argv) if (expectedHash != Hash(ht) && expectedHash != hash) throw Error(format("hash mismatch for '%1%'") % uri); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, unpack, ht); + storePath = store->addToStore(name, tmpFile, recursive, ht); - assert(*storePath == store->makeFixedOutputPath(unpack, hash, name)); + assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); } stopProgressBar(); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 806ab7563..9a26eb57f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - bool recursive = false; + FileIngestionMethod recursive = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = true; + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.empty()) @@ -194,10 +194,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - bool recursive = false; + FileIngestionMethod recursive = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = true; + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; else throw UsageError(format("unknown flag '%1%'") % i); if (opArgs.size() != 3) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 139db3657..e82ab844e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -42,10 +42,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); if (!dryRun) store->addToStore(info, sink.s); diff --git a/src/nix/command.cc b/src/nix/command.cc index 442bc6c53..fce6c391c 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -28,20 +28,20 @@ void StoreCommand::run() run(getStore()); } -StorePathsCommand::StorePathsCommand(bool recursive) +StorePathsCommand::StorePathsCommand(FileIngestionMethod recursive) : recursive(recursive) { - if (recursive) + if (recursive == FileIngestionMethod::Recursive) mkFlag() .longName("no-recursive") .description("apply operation to specified paths only") - .set(&this->recursive, false); + .set(&this->recursive, FileIngestionMethod::Flat); else mkFlag() .longName("recursive") .shortName('r') .description("apply operation to closure of the specified paths") - .set(&this->recursive, true); + .set(&this->recursive, FileIngestionMethod::Recursive); mkFlag(0, "all", "apply operation to the entire store", &all); } @@ -61,7 +61,7 @@ void StorePathsCommand::run(ref store) for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p.clone()); - if (recursive) { + if (recursive == FileIngestionMethod::Recursive) { StorePathSet closure; store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); storePaths.clear(); diff --git a/src/nix/command.hh b/src/nix/command.hh index a954a7d04..4cdda9f79 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -114,7 +114,7 @@ struct StorePathsCommand : public InstallablesCommand { private: - bool recursive = false; + FileIngestionMethod recursive = FileIngestionMethod::Flat; bool all = false; protected: @@ -123,7 +123,7 @@ protected: public: - StorePathsCommand(bool recursive = false); + StorePathsCommand(FileIngestionMethod recursive = FileIngestionMethod::Flat); using StoreCommand::run; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 85c777d38..a8ee3fce3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -17,7 +17,7 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; CmdCopy() - : StorePathsCommand(true) + : StorePathsCommand(FileIngestionMethod::Recursive) { mkFlag() .longName("from") diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f9c7fef3f..93eddbb1f 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -74,12 +74,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path.clone()); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); if (!json) printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); From f58604ac32d0ab0718a0e68b3c1b050e4ab121cb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Fri, 25 Oct 2019 17:45:14 +0200 Subject: [PATCH 036/350] Add fetchSubmodules to builtins.fetchGit There are some downsides to this features: - Submodules are not cached (unlike the root repo), - Full checkouts are created in a temporary directory. --- src/libexpr/primops/fetchGit.cc | 45 +++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..b08e48404 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,13 +22,14 @@ struct GitInfo std::string rev; std::string shortRev; uint64_t revCount = 0; + bool submodules = false; }; std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, - const std::string & name) + const std::string & name, bool fetchSubmodules) { if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -145,7 +146,8 @@ GitInfo exportGit(ref store, const std::string & uri, printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev + + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken @@ -165,16 +167,35 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - unpackTarfile(*source, tmpDir); + // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root + // repo is cached. + if (fetchSubmodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + + deletePath(tmpDir + "/.git"); + + gitInfo.submodules = true; + } else { + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); @@ -198,6 +219,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va std::optional ref; std::string rev; std::string name = "source"; + bool fetchSubmodules = false; PathSet context; state.forceValue(*args[0]); @@ -216,6 +238,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "fetchSubmodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); } @@ -230,13 +254,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules); v.attrs->sort(); if (state.allowedPaths) From c2a24c2b8876a0862ede39f63e684cb875cc7aab Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:09:50 +0200 Subject: [PATCH 037/350] Add test for fetchGit submodule support --- tests/fetchGitSubmodules.sh | 50 +++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 2 files changed, 51 insertions(+) create mode 100644 tests/fetchGitSubmodules.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh new file mode 100644 index 000000000..c821d2caa --- /dev/null +++ b/tests/fetchGitSubmodules.sh @@ -0,0 +1,50 @@ +source common.sh + +set -u + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git submodule tests" + exit 99 +fi + +clearStore + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix/gitv2 + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + + diff --git a/tests/local.mk b/tests/local.mk index dab3a23b6..01fac4fcd 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ + fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ run.sh \ From ea861be292d790db9835c08abee734baa77b9cf7 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sat, 26 Oct 2019 11:13:08 +0200 Subject: [PATCH 038/350] Add documentation for submodule support in fetchGit --- doc/manual/expressions/builtins.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 394e1fc32..838edae3c 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -422,6 +422,16 @@ stdenv.mkDerivation { … } + + fetchSubmodules + + + A boolean parameter that specifies whether submodules + should be checked out. Defaults to + false. + + + From c8d33de77781fe7850d04a7374c6c22a2e995b70 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:42:19 +0100 Subject: [PATCH 039/350] Add git submodule fixes from @bjornfor This fixes fetching repositories with no submodules and also cleans up .git files in checkouts. --- src/libexpr/local.mk | 1 + src/libexpr/primops/fetchGit.cc | 9 +++++++-- tests/fetchGitSubmodules.sh | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = + ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b08e48404..c58d28bbc 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "tarfile.hh" +#include #include #include @@ -182,9 +183,13 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - deletePath(tmpDir + "/.git"); + for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { + if (p.path().filename() == ".git") { + std::filesystem::remove_all(p.path()); + } + } gitInfo.submodules = true; } else { diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c821d2caa..987bfd69e 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -47,4 +47,5 @@ pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] - +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" From c846abb5cc900d5fcb5aeb48427a1eac646cc0f3 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:46:19 +0100 Subject: [PATCH 040/350] Add more test for git submodule functionality --- tests/fetchGitSubmodules.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 987bfd69e..c252d42f7 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -40,12 +40,23 @@ rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 435366ed3c292756bb74fd12f47ff96e93cd48ac Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:50:13 +0100 Subject: [PATCH 041/350] Rename fetchGit fetchSubmodules to just submodules --- src/libexpr/primops/fetchGit.cc | 2 +- tests/fetchGitSubmodules.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index c58d28bbc..e8905b548 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -243,7 +243,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "fetchSubmodules") + else if (n == "submodules") fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index c252d42f7..6e4ef270b 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -39,8 +39,8 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] @@ -57,6 +57,6 @@ test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; fetchSubmodules = true; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From 6c00a9545f2162496032c490aadbadd85856662c Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 15:50:39 +0100 Subject: [PATCH 042/350] Fix typo in submodule test --- tests/fetchGitSubmodules.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 6e4ef270b..9a21421cc 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -56,7 +56,8 @@ pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$roo test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. -noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; }).outPath") -noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$rev\"; submodules = true; }).outPath") +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] From cc4fe977e5eedf00d8e3d267ccbc3676b3fac1a0 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 9 Mar 2020 14:07:26 +0100 Subject: [PATCH 043/350] Link to stdc++fs Some platforms seem to still require linking with stdc++fs to enable STL std::filesystem support. --- src/libexpr/local.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9b5fcc561..b6ee82424 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,11 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust +ifeq ($(CXX), g++) +libexpr_LDFLAGS = -lstdc++fs +else libexpr_LDFLAGS = +endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl From 002a3a95dcb7391a8372ba7a74f4e7e9bd48b59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:21 +0100 Subject: [PATCH 044/350] fetchGit: fix "fatal: couldn't find remote ref refs/heads/master" issue with submodules --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e8905b548..66259d778 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -179,7 +179,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + "--", cacheDir, fmt("%s", *ref) }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 587e259bfdda248fb2d81389faca816b8efdb5f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:55 +0100 Subject: [PATCH 045/350] tests/fetchGitSubmodules: add more tests --- tests/fetchGitSubmodules.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 9a21421cc..8ee1bb067 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -42,6 +42,25 @@ pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootR pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 6864ad7cf510d96f0cfc0cdbeffc1188a4eab44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:14 +0100 Subject: [PATCH 046/350] fetchGit: fix submodule output attribute Before this change it would be false for all evaluations but the first. Now it follows the input argument (as it should). --- src/libexpr/primops/fetchGit.cc | 7 +++---- tests/fetchGitSubmodules.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66259d778..653d99b53 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -32,6 +32,9 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) { + GitInfo gitInfo; + gitInfo.submodules = fetchSubmodules; + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -49,7 +52,6 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -141,7 +143,6 @@ GitInfo exportGit(ref store, const std::string & uri, } // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -190,8 +191,6 @@ GitInfo exportGit(ref store, const std::string & uri, std::filesystem::remove_all(p.path()); } } - - gitInfo.submodules = true; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 8ee1bb067..2d625c376 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -61,6 +61,15 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo [[ $r6 == $r7 ]] [[ $r7 == $r8 ]] +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] From 369fffd6f1f7d939e528b3591dca49989c51ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:58 +0100 Subject: [PATCH 047/350] fetchGit: add submodules attribute to the .link file The .link file is used as a lock, so I think we should put the "submodule" attribute in there since turning on submodules creates a new .link file path. --- src/libexpr/primops/fetchGit.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 653d99b53..969b1a4ed 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -211,6 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, json["name"] = name; json["rev"] = gitInfo.rev; json["revCount"] = gitInfo.revCount; + json["submodules"] = gitInfo.submodules; writeFile(storeLink, json.dump()); From be84049baf047fd2eee08def5c8013797c0832a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 20:14:19 +0100 Subject: [PATCH 048/350] tests/fetchGitSubmodules.sh: more checks --- tests/fetchGitSubmodules.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh index 2d625c376..30a8737bd 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/fetchGitSubmodules.sh @@ -38,10 +38,6 @@ git -C $rootRepo commit -m "Add submodule" rev=$(git -C $rootRepo rev-parse HEAD) -pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") -pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") - r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") @@ -70,16 +66,26 @@ have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\" have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") [[ $have_submodules == true ]] +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"refs/heads/master\"; rev = \"$rev\"; submodules = true; }).outPath") + # The resulting store path cannot be the same. [[ $pathWithoutSubmodules != $pathWithSubmodules ]] # Checking out the same repo with submodules returns in the same store path. [[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + # The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] [[ -e $pathWithSubmodules/sub/content ]] +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + # No .git directory or submodule reference files must be left test "$(find "$pathWithSubmodules" -name .git)" = "" From b306b7039ef3f2b76dbcc8020dcf67ed27442e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:44:37 +0100 Subject: [PATCH 049/350] fetchGit: checkout rev instead of latest ref Major bugfix for the submodules = true code path. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 969b1a4ed..0c6bc6dc0 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -182,7 +182,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--", cacheDir, fmt("%s", *ref) }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); From cc522d0d23b71faeac95998d264abe14ddceeed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:45:31 +0100 Subject: [PATCH 050/350] fetchGit: fix submodules = true for dirty trees --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 0c6bc6dc0..66e1d7b98 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -55,8 +55,12 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" }); + if (fetchSubmodules) { + gitOpts.emplace_back("--recurse-submodules"); + } auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); + runProgram("git", true, gitOpts), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, uri)); From f686efeed4d6305101c56136855e2d6b87e649e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Fri, 20 Mar 2020 14:36:10 +0100 Subject: [PATCH 051/350] fetchGit: fix submodule corner case by fetching all refs from cacheDir Due to fetchGit not checking if rev is an ancestor of ref (there is even a FIXME comment about it in the code), the cache repo might not have the ref even though it has the rev. This doesn't matter when submodule = false, but the submodule = true code blows up because it tries to fetch the (missing) ref from the cache repo. Fix this in the simplest way possible: fetch all refs from the local cache repo when submodules = true. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66e1d7b98..f138b755c 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -183,8 +183,11 @@ GitInfo exportGit(ref store, const std::string & uri, AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: the cacheDir repo might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch everything to + // ensure we get the rev. runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s", *ref) }); + "--update-head-ok", "--", cacheDir, "refs/*:refs/*" }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); From 40c023ecfe49fea6e66db34c5f841fcf7001cbeb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sun, 29 Mar 2020 23:47:48 +0200 Subject: [PATCH 052/350] fetchGit: don't use std::filesystem to filter git repos Using std::filesystem means also having to link with -lstdc++fs on some platforms and it's hard to discover for what platforms this is needed. As all the functionality is already implemented as utilities, use those instead. --- src/libexpr/local.mk | 4 ---- src/libexpr/primops/fetchGit.cc | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b6ee82424..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,11 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust -ifeq ($(CXX), g++) -libexpr_LDFLAGS = -lstdc++fs -else libexpr_LDFLAGS = -endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f138b755c..d60e6b803 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,7 +6,6 @@ #include "hash.hh" #include "tarfile.hh" -#include #include #include @@ -28,6 +27,13 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) @@ -175,6 +181,7 @@ GitInfo exportGit(ref store, const std::string & uri, Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root // repo is cached. @@ -193,11 +200,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { - if (p.path().filename() == ".git") { - std::filesystem::remove_all(p.path()); - } - } + filter = isNotDotGitDirectory; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); @@ -208,7 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, unpackTarfile(*source, tmpDir); } - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); + gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter)); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); From e322a16523c3c0644a900453abbe32bd0fd982e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:29:29 +0200 Subject: [PATCH 053/350] Remove global -I flags (cherry picked from commit 2c692a3b144523bca68dd6de618124ba6c9bb332) --- local.mk | 4 +++- mk/precompiled-headers.mk | 4 ++-- src/build-remote/build-remote.cc | 2 +- src/libexpr/local.mk | 2 ++ src/libexpr/primops/fromTOML.cc | 2 +- src/libmain/local.mk | 2 ++ src/libstore/local.mk | 3 ++- src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-copy-closure/nix-copy-closure.cc | 2 +- src/nix-daemon/nix-daemon.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/local.mk | 2 ++ tests/plugins/local.mk | 2 ++ 18 files changed, 27 insertions(+), 16 deletions(-) diff --git a/local.mk b/local.mk index 55f85a044..d254c10fe 100644 --- a/local.mk +++ b/local.mk @@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(GCH) $(PCH): src/libutil/util.hh config.h + +GCH_CXXFLAGS = -I src/libutil diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1a727ba1b..1c0452dc2 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) PCH = $(buildprefix)precompiled-headers.h.pch $(PCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) $(PCH) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 69d1c6f7e..00340b787 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,7 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using std::cin; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..a4ccab376 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,6 +6,8 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr + libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index a84e569e9..c43324dbb 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,7 +1,7 @@ #include "primops.hh" #include "eval-inline.hh" -#include "cpptoml/cpptoml.h" +#include "../../cpptoml/cpptoml.h" namespace nix { diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 0c80f5a0a..a8eed6c65 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -6,6 +6,8 @@ libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +libmain_CXXFLAGS += -I src/libutil -I src/libstore + libmain_LDFLAGS = $(OPENSSL_LIBS) libmain_LIBS = libstore libutil diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..91acef368 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif -libstore_CXXFLAGS = \ +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 1bda159d0..0a058a31b 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,7 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using namespace std::string_literals; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 1b337a712..a2639579d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "globals.hh" #include "download.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index d4060ac93..aa5ada3a6 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,7 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index f87035760..b10184718 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 134898561..e68d1b1be 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "derivations.hh" #include "finally.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include "daemon.hh" #include diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index e47b00acf..1a2bb42a3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,7 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 617d927a4..6c99d1181 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 18ced94b1..2b9254659 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -6,9 +6,9 @@ #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" #include "finally.hh" -#include "progress-bar.hh" +#include "../nix/legacy.hh" +#include "../nix/progress-bar.hh" #include "tarfile.hh" #include diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 806ab7563..fcc00175a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "graphml.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include #include diff --git a/src/nix/local.mk b/src/nix/local.mk index 51dad101f..50a18efd7 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,6 +15,8 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain + nix_LIBS = libexpr libmain libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 1d2bac052..82ad99402 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -7,3 +7,5 @@ libplugintest_SOURCES := $(d)/plugintest.cc libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 + +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr From 367577d9a6ec465d773f7b3ba53a5656253f514d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 17:00:40 +0200 Subject: [PATCH 054/350] Fix macOS build --- src/resolve-system-dependencies/local.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index bf65f7905..f0e82e023 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,8 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix +resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain + resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc From 35c7bab09ac97efd3561d3c5a0c827e4dc391141 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 30 Mar 2020 09:14:29 -0600 Subject: [PATCH 055/350] build with make --- Makefile | 3 ++- src/libutil/util.hh | 1 + tests/errors/local.mk | 17 +++++++++++++++++ tests/errors/main.cc | 6 ++++-- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 tests/errors/local.mk diff --git a/Makefile b/Makefile index 469070533..2d70aeb27 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,8 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk + tests/plugins/local.mk \ + tests/errors/local.mk -include Makefile.config diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d69e29158..0b55c6788 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "logging.hh" #include "ansicolor.hh" +#include "error.hh" #include #include diff --git a/tests/errors/local.mk b/tests/errors/local.mk new file mode 100644 index 000000000..c5b9cc5c8 --- /dev/null +++ b/tests/errors/local.mk @@ -0,0 +1,17 @@ +programs += error-test + +error-test_DIR := $(d) + +error-test_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +error-test_LIBS = libutil + +error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system + +# $(foreach name, \ +# nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ +# $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +# $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) + +# src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 35d2a1cdf..39adc2ece 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -12,7 +12,7 @@ int main() { using namespace nix; - ErrorInfo::programName = optional("errorTest"); + ErrorInfo::programName = optional("error-test"); /* ColumnRange columnRange; @@ -59,7 +59,9 @@ int main() .columnRange(13,7) .linesOfCode(nullopt ,"this is the problem line of code" - ,nullopt)); + ,nullopt) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); printErrorInfo(MkNixError() .name("error name") From 28d073e810d5e692a4f6c57d23afc011e6fcc153 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 30 Mar 2020 09:15:21 -0600 Subject: [PATCH 056/350] remove cruft --- tests/errors/local.mk | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/errors/local.mk b/tests/errors/local.mk index c5b9cc5c8..0d64c8dbd 100644 --- a/tests/errors/local.mk +++ b/tests/errors/local.mk @@ -8,10 +8,3 @@ error-test_SOURCES := \ error-test_LIBS = libutil error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system - -# $(foreach name, \ -# nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ -# $(eval $(call install-symlink, nix, $(bindir)/$(name)))) -# $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) - -# src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh From e1a94ad852e91fcdcf4efb60fe7e9b9e328df7ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 19:14:17 +0200 Subject: [PATCH 057/350] Backport 'nix dev-shell' from the flakes branch This also adds a '--profile' option to 'nix build' (replacing 'nix-env --set'). --- src/libutil/util.cc | 11 ++ src/libutil/util.hh | 12 +- src/nix/build.cc | 17 ++- src/nix/command.cc | 94 ++++++++++++ src/nix/command.hh | 62 ++++---- src/nix/installables.cc | 5 + src/nix/installables.hh | 45 ++++++ src/nix/run.cc | 120 ++++++--------- src/nix/shell.cc | 319 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 569 insertions(+), 116 deletions(-) create mode 100644 src/nix/installables.hh create mode 100644 src/nix/shell.cc diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 097ff210a..332c1c43a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -478,6 +478,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 7c3a30242..1f85c7c46 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -122,10 +122,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - std::string getUserName(); /* Return $HOME or the user's home directory from /etc/passwd. */ @@ -205,6 +201,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair createTempFile(const Path & prefix = "nix"); + + class Pipe { public: diff --git a/src/nix/build.cc b/src/nix/build.cc index 3c9d2df39..0b0762836 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,7 +5,7 @@ using namespace nix; -struct CmdBuild : MixDryRun, InstallablesCommand +struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; @@ -40,6 +40,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand "To build the build.x86_64-linux attribute from release.nix:", "nix build -f release.nix build.x86_64-linux" }, + Example{ + "To make a profile point at GNU Hello:", + "nix build --profile /tmp/profile nixpkgs.hello" + }, }; } @@ -49,18 +53,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand if (dryRun) return; - for (size_t i = 0; i < buildables.size(); ++i) { - auto & b(buildables[i]); - - if (outLink != "") - for (auto & output : b.outputs) + if (outLink != "") { + for (size_t i = 0; i < buildables.size(); ++i) { + for (auto & output : buildables[i].outputs) if (auto store2 = store.dynamic_pointer_cast()) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); store2->addPermRoot(output.second, absPath(symlink), true); } + } } + + updateProfile(buildables); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index 442bc6c53..99b24d2a2 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -2,6 +2,9 @@ #include "store-api.hh" #include "derivations.hh" #include "nixexpr.hh" +#include "profiles.hh" + +extern char * * environ; namespace nix { @@ -96,4 +99,95 @@ Strings editorFor(const Pos & pos) return args; } +MixProfile::MixProfile() +{ + mkFlag() + .longName("profile") + .description("profile to update") + .labels({"path"}) + .dest(&profile); +} + +void MixProfile::updateProfile(const StorePath & storePath) +{ + if (!profile) return; + auto store = getStore().dynamic_pointer_cast(); + if (!store) throw Error("'--profile' is not supported for this Nix store"); + auto profile2 = absPath(*profile); + switchLink(profile2, + createGeneration( + ref(store), + profile2, store->printStorePath(storePath))); +} + +void MixProfile::updateProfile(const Buildables & buildables) +{ + if (!profile) return; + + std::optional result; + + for (auto & buildable : buildables) { + for (auto & output : buildable.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second.clone(); + } + } + + if (!result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + + updateProfile(*result); +} + +MixDefaultProfile::MixDefaultProfile() +{ + profile = getDefaultProfile(); +} + +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { + mkFlag() + .longName("ignore-environment") + .shortName('i') + .description("clear the entire environment (except those specified with --keep)") + .set(&ignoreEnvironment, true); + + mkFlag() + .longName("keep") + .shortName('k') + .description("keep specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { keep.insert(ss.front()); }); + + mkFlag() + .longName("unset") + .shortName('u') + .description("unset specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { unset.insert(ss.front()); }); +} + +void MixEnvironment::setEnviron() { + if (ignoreEnvironment) { + if (!unset.empty()) + throw UsageError("--unset does not make sense with --ignore-environment"); + + for (const auto & var : keep) { + auto val = getenv(var.c_str()); + if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val)); + } + + vectorEnv = stringsToCharPtrs(stringsEnv); + environ = vectorEnv.data(); + } else { + if (!keep.empty()) + throw UsageError("--keep does not make sense without --ignore-environment"); + + for (const auto & var : unset) + unsetenv(var.c_str()); + } +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index a954a7d04..23f5c9898 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,5 +1,6 @@ #pragma once +#include "installables.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -22,34 +23,7 @@ private: std::shared_ptr _store; }; -struct Buildable -{ - std::optional drvPath; - std::map outputs; -}; - -typedef std::vector Buildables; - -struct Installable -{ - virtual ~Installable() { } - - virtual std::string what() = 0; - - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } - - Buildable toBuildable(); - - virtual std::pair toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } -}; - -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +struct SourceExprCommand : virtual StoreCommand, MixEvalArgs { Path file; @@ -184,4 +158,36 @@ std::set toDerivations(ref store, filename:lineno. */ Strings editorFor(const Pos & pos); +struct MixProfile : virtual StoreCommand +{ + std::optional profile; + + MixProfile(); + + /* If 'profile' is set, make it point at 'storePath'. */ + void updateProfile(const StorePath & storePath); + + /* If 'profile' is set, make it point at the store path produced + by 'buildables'. */ + void updateProfile(const Buildables & buildables); +}; + +struct MixDefaultProfile : MixProfile +{ + MixDefaultProfile(); +}; + +struct MixEnvironment : virtual Args { + + StringSet keep, unset; + Strings stringsEnv; + std::vector vectorEnv; + bool ignoreEnvironment; + + MixEnvironment(); + + /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + void setEnviron(); +}; + } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 013218cd9..f464d0aa1 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -109,6 +109,11 @@ struct InstallableStorePath : Installable bs.push_back(std::move(b)); return bs; } + + std::optional getStorePath() override + { + return storePath.clone(); + } }; struct InstallableValue : Installable diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000..503984220 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "util.hh" +#include "path.hh" +#include "eval.hh" + +#include + +namespace nix { + +struct Buildable +{ + std::optional drvPath; + std::map outputs; +}; + +typedef std::vector Buildables; + +struct Installable +{ + virtual ~Installable() { } + + virtual std::string what() = 0; + + virtual Buildables toBuildables() + { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + virtual std::pair toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } + + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional getStorePath() + { + return {}; + } +}; + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index f885c5e49..8e30264c0 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -8,6 +8,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "affinity.hh" +#include "eval.hh" #if __linux__ #include @@ -19,11 +20,46 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct CmdRun : InstallablesCommand +struct RunCommon : virtual Command +{ + void runProgram(ref store, + const std::string & program, + const Strings & args) + { + stopProgressBar(); + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); + } +}; + +struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment { std::vector command = { "bash" }; - StringSet keep, unset; - bool ignoreEnvironment = false; CmdRun() { @@ -37,28 +73,6 @@ struct CmdRun : InstallablesCommand if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; }); - - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); - - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { keep.insert(ss.front()); }); - - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { unset.insert(ss.front()); }); } std::string description() override @@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand auto accessor = store->getFSAccessor(); - if (ignoreEnvironment) { - - if (!unset.empty()) - throw UsageError("--unset does not make sense with --ignore-environment"); - - std::map kept; - for (auto & var : keep) { - auto s = getenv(var.c_str()); - if (s) kept[var] = s; - } - - clearEnv(); - - for (auto & var : kept) - setenv(var.first.c_str(), var.second.c_str(), 1); - - } else { - - if (!keep.empty()) - throw UsageError("--keep does not make sense without --ignore-environment"); - - for (auto & var : unset) - unsetenv(var.c_str()); - } std::unordered_set done; std::queue todo; for (auto & path : outPaths) todo.push(path.clone()); + setEnviron(); + auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { @@ -142,38 +134,10 @@ struct CmdRun : InstallablesCommand setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - std::string cmd = *command.begin(); Strings args; for (auto & arg : command) args.push_back(arg); - stopProgressBar(); - - restoreSignals(); - - restoreAffinity(); - - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast(); - - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; - for (auto & arg : args) helperArgs.push_back(arg); - - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - - throw SysError("could not execute chroot helper"); - } - - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("unable to exec '%s'", cmd); + runProgram(store, *command.begin(), args); } }; diff --git a/src/nix/shell.cc b/src/nix/shell.cc new file mode 100644 index 000000000..82b1da552 --- /dev/null +++ b/src/nix/shell.cc @@ -0,0 +1,319 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +struct Var +{ + bool exported; + std::string value; // quoted string or array +}; + +struct BuildEnvironment +{ + std::map env; + std::string bashFunctions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + std::set exported; + + debug("reading environment file '%s'", path); + + auto file = readFile(path); + + auto pos = file.cbegin(); + + static std::string varNameRegex = + R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + + static std::regex declareRegex( + "^declare -x (" + varNameRegex + ")" + + R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + + static std::string simpleStringRegex = + R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + + static std::string quotedStringRegex = + R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + + static std::string arrayRegex = + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + + static std::regex varRegex( + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + + static std::regex functionRegex( + "^" + varNameRegex + " \\(\\) *\n"); + + while (pos != file.end()) { + + std::smatch match; + + if (std::regex_search(pos, file.cend(), match, declareRegex)) { + pos = match[0].second; + exported.insert(match[1]); + } + + else if (std::regex_search(pos, file.cend(), match, varRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + res.bashFunctions = std::string(pos, file.cend()); + break; + } + + else throw Error("shell environment '%s' has unexpected line '%s'", + path, file.substr(pos - file.cbegin(), 60)); + } + + return res; +} + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +StorePath getDerivationEnvironment(ref store, Derivation drv) +{ + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + + drv.args = { + "-c", + "set -e; " + "export IN_NIX_SHELL=impure; " + "export dontAddDisableDepTrack=1; " + "if [[ -n $stdenv ]]; then " + " source $stdenv/setup; " + "fi; " + "export > $out; " + "set >> $out "}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + // FIXME: handle structured attrs + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = drv.env["name"] + "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + Hash h = hashDerivationModulo(*store, drv, true); + auto shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); + drv.env["out"] = store->printStorePath(shellOutPath); + auto shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return shellOutPath; +} + +struct Common : InstallableCommand, MixProfile +{ + std::set ignoreVars{ + "BASHOPTS", + "EUID", + "HOME", // FIXME: don't ignore in pure mode? + "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", + "NIX_LOG_FD", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt + "TEMP", + "TEMPDIR", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + out << "nix_saved_PATH=\"$PATH\"\n"; + + for (auto & i : buildEnvironment.env) { + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } + } + + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + + out << buildEnvironment.bashFunctions << "\n"; + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; + } + + StorePath getShellOutPath(ref store) + { + auto path = installable->getStorePath(); + if (path && hasSuffix(path->to_string(), "-env")) + return path->clone(); + else { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + } + } + + BuildEnvironment getBuildEnvironment(ref store) + { + auto shellOutPath = getShellOutPath(store); + + updateProfile(shellOutPath); + + return readEnvironment(store->printStorePath(shellOutPath)); + } +}; + +struct CmdDevShell : Common, MixEnvironment +{ + std::vector command; + + CmdDevShell() + { + mkFlag() + .longName("command") + .shortName('c') + .description("command and arguments to be executed insted of an interactive shell") + .labels({"command", "args"}) + .arity(ArityAny) + .handler([&](std::vector ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }); + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix dev-shell nixpkgs.hello" + }, + Example{ + "To store the build environment in a profile:", + "nix dev-shell --profile /tmp/my-shell nixpkgs.hello" + }, + Example{ + "To use a build environment previously recorded in a profile:", + "nix dev-shell /tmp/my-shell" + }, + }; + } + + void run(ref store) override + { + auto buildEnvironment = getBuildEnvironment(store); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + if (!command.empty()) { + std::vector args; + for (auto s : command) + args.push_back(shellEscape(s)); + ss << fmt("exec %s\n", concatStringsSep(" ", args)); + } + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL").value_or("bash"); + + setEnviron(); + + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs.hello)" + }, + }; + } + + void run(ref store) override + { + auto buildEnvironment = getBuildEnvironment(store); + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static auto r1 = registerCommand("print-dev-env"); +static auto r2 = registerCommand("dev-shell"); From bbbb7c1bc7a66e1169931cda8bd863b68236726a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2020 18:15:55 -0400 Subject: [PATCH 058/350] Use `auto` with some `FileIngestionMethod` local variables --- src/libexpr/primops.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7d45733f4..71a511c41 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional outputHash; std::string outputHashAlgo; - FileIngestionMethod outputHashRecursive = FileIngestionMethod::Flat; + auto outputHashRecursive = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9a26eb57f..cde7d561b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -174,7 +174,7 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - FileIngestionMethod recursive = FileIngestionMethod::Flat; + auto recursive = FileIngestionMethod::Flat; for (auto & i : opFlags) if (i == "--recursive") recursive = FileIngestionMethod::Recursive; @@ -194,7 +194,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - FileIngestionMethod recursive = FileIngestionMethod::Flat; + auto recursive = FileIngestionMethod::Flat; for (auto i : opFlags) if (i == "--recursive") recursive = FileIngestionMethod::Recursive; From 51afea3af23640a3fb2cfe8cfae5cf3cc27eb3a1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2020 22:31:51 +0000 Subject: [PATCH 059/350] Never cast `FileIngestionMethod` to or from boolean --- src/libexpr/primops.cc | 9 ++++++--- src/libstore/build.cc | 10 +++++++--- src/libstore/path.hh | 2 +- src/libstore/store-api.cc | 21 ++++++++++++++------- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 71a511c41..4d5aaf86a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -724,9 +724,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), - (static_cast(outputHashRecursive) ? "r:" : "") + printHashType(h.type), - h.to_string(Base16, false))); + drv.outputs.insert_or_assign("out", DerivationOutput { + std::move(outPath), + (outputHashRecursive == FileIngestionMethod::Recursive ? "r:" : "") + + printHashType(h.type), + h.to_string(Base16, false), + }); } else { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 224633106..be52e637e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3650,7 +3650,7 @@ void DerivationGoal::registerOutputs() FileIngestionMethod recursive; Hash h; i.second.parseHashInfo(recursive, h); - if (!static_cast(recursive)) { + if (recursive == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3659,7 +3659,9 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = static_cast(recursive) ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = recursive == FileIngestionMethod::Recursive + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); @@ -3912,7 +3914,9 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - auto used = static_cast(recursive) ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); + auto used = recursive + ? cloneStorePathSet(getClosure(info.path).first) + : cloneStorePathSet(info.references); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 73c4b8e29..5122e7422 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -73,7 +73,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -enum struct FileIngestionMethod : bool { +enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06aa0883c..ca1409588 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -177,13 +177,16 @@ StorePath Store::makeFixedOutputPath( const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (static_cast(recursive) ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), name); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + hash.to_string(Base16) + ":"), + name); } } @@ -202,7 +205,9 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const { - Hash h = static_cast(recursive) ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); + Hash h = recursive == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); return std::make_pair(makeFixedOutputPath(recursive, h, name), h); } @@ -782,7 +787,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, static_cast(recursive) ? 8 : 6)); + Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { @@ -828,7 +833,9 @@ Strings ValidPathInfo::shortRefs() const std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { - return "fixed:" + (static_cast(recursive) ? (std::string) "r:" : "") + hash.to_string(); + return "fixed:" + + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + hash.to_string(); } From 7e9a2718f0c39fde53a3447c3b7f3617a4a9b958 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2020 22:36:15 +0000 Subject: [PATCH 060/350] s/outputHashRecursive/ingestionMethod/c --- src/libexpr/primops.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4d5aaf86a..415e77fe7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -563,7 +563,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional outputHash; std::string outputHashAlgo; - auto outputHashRecursive = FileIngestionMethod::Flat; + auto ingestionMethod = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); @@ -574,8 +574,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { - if (s == "recursive") outputHashRecursive = FileIngestionMethod::Recursive; - else if (s == "flat") outputHashRecursive = FileIngestionMethod::Flat; + if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; + else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); }; @@ -722,11 +722,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { std::move(outPath), - (outputHashRecursive == FileIngestionMethod::Recursive ? "r:" : "") + (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") + printHashType(h.type), h.to_string(Base16, false), }); From 8aa46cd340c1294c3d06cd52f85c906bdf749070 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Mar 2020 22:40:41 +0000 Subject: [PATCH 061/350] Get rid of FileIngestionMethod casts in perl bindings, too --- perl/lib/Nix/Store.xs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index eab8ccacb..890310b3e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, (FileIngestionMethod) recursive, parseHashType(algo)); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { Hash h(hash, parseHashType(algo)); - auto path = store()->makeFixedOutputPath((FileIngestionMethod) recursive, h, name); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->makeFixedOutputPath(method, h, name); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 12556e570924315eb25ad6057b6c2c5162955e4f Mon Sep 17 00:00:00 2001 From: mlatus Date: Tue, 31 Mar 2020 19:40:16 +0800 Subject: [PATCH 062/350] fix placeholder not substituted in passAsFile --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e3a23a4d..527d7ac42 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2465,7 +2465,7 @@ void DerivationGoal::initTmpDir() { auto hash = hashString(htSHA256, i.first); string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; - writeFile(p, i.second); + writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } From 3166b97174ab879dac3b19eb3f257e73a943fef0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 13:45:28 +0200 Subject: [PATCH 063/350] nix shell -> nix dev-shell --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 82b1da552..71e640667 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -92,7 +92,7 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) { auto builder = baseNameOf(drv.builder); if (builder != "bash") - throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); drv.args = { "-c", From 09652f597cb44fbb452353f9eeaba2d9bc70a7f3 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 31 Mar 2020 09:36:20 -0600 Subject: [PATCH 064/350] enum style --- src/libutil/error.hh | 4 ++-- tests/errors/main.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 88980afb7..1acc70d9d 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,10 +16,10 @@ using std::endl; namespace nix { -enum ErrLevel +typedef enum { elWarning , elError - }; + } ErrLevel; class ColumnRange { public: diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 39adc2ece..ffce334b7 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -40,7 +40,7 @@ int main() printErrorInfo(StandardError() .name("name") - .description("description") + .description("error description") .nohint() ); From 9e7b89bf100500cf252f785e2dd77f83ee5cf7a4 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 31 Mar 2020 11:56:37 -0600 Subject: [PATCH 065/350] rename errors/warnings --- src/libutil/error.hh | 8 ++++---- tests/errors/main.cc | 32 ++++---------------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1acc70d9d..cd2b2832d 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -256,11 +256,11 @@ class AddHint : private T typedef AddName< AddDescription< AddHint< - EIError>>> StandardError; + EIError>>> ProgramError; typedef AddName< AddDescription< AddHint< - EIWarning>>> StandardWarning; + EIWarning>>> ProgramWarning; typedef AddName< AddDescription< @@ -269,7 +269,7 @@ typedef AddName< AddColumnRange< AddLOC< AddHint< - EIError>>>>>>> MkNixError; + EIError>>>>>>> NixLangError; typedef AddName< AddDescription< AddNixFile< @@ -277,7 +277,7 @@ typedef AddName< AddColumnRange< AddLOC< AddHint< - EIWarning>>>>>>> MkNixWarning; + EIWarning>>>>>>> NixLangWarning; // -------------------------------------------------------- diff --git a/tests/errors/main.cc b/tests/errors/main.cc index ffce334b7..101d44a7e 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -14,44 +14,20 @@ int main() ErrorInfo::programName = optional("error-test"); - /* - ColumnRange columnRange; - columnRange.start = 24; - columnRange.len = 14; - - ErrLine errline; - errline.lineNumber = 7; - errline.columnRange = optional(columnRange); - errline.errLineOfCode = "line of code where the error occurred"; - - NixCode nixcode; - nixcode.nixFile = optional("myfile.nix"); - nixcode.errLine = errline; - - ErrorInfo generic; - generic.level = elError; - generic.name = "error name"; - generic.description = "general error description"; - generic.program = "nixtool.exe"; - generic.nixCode = nixcode; - - printErrorInfo(generic); - */ - - printErrorInfo(StandardError() + printErrorInfo(ProgramError() .name("name") .description("error description") .nohint() ); - printErrorInfo(StandardWarning() + printErrorInfo(ProgramWarning() .name("warning name") .description("warning description") .nohint() ); - printErrorInfo(MkNixWarning() + printErrorInfo(NixLangWarning() .name("warning name") .description("warning description") .nixFile("myfile.nix") @@ -63,7 +39,7 @@ int main() .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") ); - printErrorInfo(MkNixError() + printErrorInfo(NixLangError() .name("error name") .description("error description") .nixFile("myfile.nix") From 5b3aefff857d038d49ad9a5d92371b4f6b19e8b9 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 31 Mar 2020 12:42:41 -0600 Subject: [PATCH 066/350] add some explanatory comments --- src/libutil/error.cc | 4 ++- src/libutil/error.hh | 2 ++ tests/errors/main.cc | 83 +++++++++++++++++++++++++++----------------- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c986affcc..5384248f8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -152,12 +152,14 @@ void printErrorInfo(ErrorInfo &einfo) // lines of code. if (einfo.nixCode.has_value()) + { printCodeLines(prefix, *einfo.nixCode); + cout << prefix << endl; + } // hint if (einfo.hint.has_value()) { - cout << prefix << endl; cout << prefix << *einfo.hint << endl; cout << prefix << endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index cd2b2832d..00539f06a 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -257,6 +257,7 @@ typedef AddName< AddDescription< AddHint< EIError>>> ProgramError; + typedef AddName< AddDescription< AddHint< @@ -270,6 +271,7 @@ typedef AddName< AddLOC< AddHint< EIError>>>>>>> NixLangError; + typedef AddName< AddDescription< AddNixFile< diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 101d44a7e..1aff5a016 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -8,48 +8,67 @@ using std::nullopt; using std::cout; using std::endl; + int main() { using namespace nix; + // In each program where errors occur, this has to be set. ErrorInfo::programName = optional("error-test"); - printErrorInfo(ProgramError() - .name("name") - .description("error description") - .nohint() - ); + // There are currently four error types - + // ProgramError, ProgramWarning, NixLangError, NixLangWarning. + // Each error type is created with a specific sequence of builder functions. + // Unlike with a constructor, each parameter is clearly named. + // If the sequence of function calls isn't followed, then there's a type error. + // This should make for a consistent look in the code when errors are created. - printErrorInfo(ProgramWarning() - .name("warning name") - .description("warning description") - .nohint() - ); + // ProgramError takes name, description, and an optional hint. + printErrorInfo( + ProgramError() + .name("name") + .description("error description") + .nohint() + ); + // ProgramWarning takes name, description, and an optional hint. + // The hint is in the form of a hintfmt class, which wraps boost::format(), and + // makes all the substituted text yellow. + printErrorInfo( + ProgramWarning() + .name("warning name") + .description("warning description") + .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow. + ); - printErrorInfo(NixLangWarning() - .name("warning name") - .description("warning description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(nullopt - ,"this is the problem line of code" - ,nullopt) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangWarning adds nix file, line number, column range, and the lines of code + // where a warning occurred. + printErrorInfo( + NixLangWarning() + .name("warning name") + .description("warning description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(nullopt + ,"this is the problem line of code" + ,nullopt) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); - printErrorInfo(NixLangError() - .name("error name") - .description("error description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(optional("previous line of code") - ,"this is the problem line of code" - ,optional("next line of code")) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangError is just the same as NixLangWarning, except for the Error flag. + printErrorInfo( + NixLangError() + .name("error name") + .description("error description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(optional("previous line of code") + ,"this is the problem line of code" + ,optional("next line of code")) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); return 0; } From a72b6b2ec80a8500015315acd36991aad9817907 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 31 Mar 2020 18:29:41 -0600 Subject: [PATCH 067/350] examples of invalid errors --- tests/errors/main.cc | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 1aff5a016..9d08e82ea 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -16,8 +16,10 @@ int main() // In each program where errors occur, this has to be set. ErrorInfo::programName = optional("error-test"); - // There are currently four error types - + // There are currently four error types: + // // ProgramError, ProgramWarning, NixLangError, NixLangWarning. + // // Each error type is created with a specific sequence of builder functions. // Unlike with a constructor, each parameter is clearly named. // If the sequence of function calls isn't followed, then there's a type error. @@ -41,6 +43,27 @@ int main() .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow. ); + /* + // some invalid errors: + + // type error: no hint function. + ProgramError() + .name("name") + .description("error description"); + + // type error: description before name. + ProgramError() + .description("error description") + .name("name") + .nohint(); + + // type error: hint function with regular boost format, not special hintfmt. + ProgramError() + .description("error description") + .name("name") + .hint(format("there was a %1%") % "warning"); + */ + // NixLangWarning adds nix file, line number, column range, and the lines of code // where a warning occurred. printErrorInfo( From 8713aeac5e29f019286df6dc6c52a2da956972e7 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 1 Apr 2020 15:51:14 -0600 Subject: [PATCH 068/350] remove using std::*, switch to include guard --- src/libutil/error.cc | 6 +- src/libutil/error.hh | 14 ++-- tests/errors/main.cc | 150 +++++++++++++++++++++---------------------- 3 files changed, 78 insertions(+), 92 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 5384248f8..2b7a079dc 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -5,11 +5,7 @@ namespace nix { -using std::cout; -using std::endl; -using std::nullopt; - -optional ErrorInfo::programName = nullopt; +optional ErrorInfo::programName = std::nullopt; // return basic_format? string showErrLine(ErrLine &errLine) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 00539f06a..614f08a35 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,4 +1,5 @@ -#pragma once +#ifndef error_hh +#define error_hh #include "ansicolor.hh" #include @@ -8,12 +9,6 @@ #include -using std::string; -using std::optional; -using boost::format; -using std::cout; -using std::endl; - namespace nix { typedef enum @@ -36,7 +31,6 @@ class ErrLine { optional prevLineOfCode; string errLineOfCode; optional nextLineOfCode; - }; class NixCode { @@ -112,8 +106,6 @@ class ErrorInfo { protected: // constructor is protected, so only the builder classes can create an ErrorInfo. ErrorInfo(ErrLevel level) { this->level = level; } - - }; // Init as error @@ -285,7 +277,9 @@ typedef AddName< // -------------------------------------------------------- // error printing +// just to cout for now. void printErrorInfo(ErrorInfo &einfo); } +#endif diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 9d08e82ea..f11390c73 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -3,95 +3,91 @@ #include #include -using std::optional; -using std::nullopt; -using std::cout; -using std::endl; int main() { - using namespace nix; + using namespace nix; - // In each program where errors occur, this has to be set. - ErrorInfo::programName = optional("error-test"); + // In each program where errors occur, this has to be set. + ErrorInfo::programName = optional("error-test"); - // There are currently four error types: - // - // ProgramError, ProgramWarning, NixLangError, NixLangWarning. - // - // Each error type is created with a specific sequence of builder functions. - // Unlike with a constructor, each parameter is clearly named. - // If the sequence of function calls isn't followed, then there's a type error. - // This should make for a consistent look in the code when errors are created. + // There are currently four error types: + // + // ProgramError, ProgramWarning, NixLangError, NixLangWarning. + // + // Each error type is created with a specific sequence of builder functions. + // Unlike with a constructor, each parameter is clearly named. + // If the sequence of function calls isn't followed, then there's a type error. + // This should make for a consistent look in the code when errors are created. - // ProgramError takes name, description, and an optional hint. - printErrorInfo( - ProgramError() - .name("name") - .description("error description") - .nohint() - ); + // ProgramError takes name, description, and an optional hint. + printErrorInfo( + ProgramError() + .name("name") + .description("error description") + .nohint() + ); - // ProgramWarning takes name, description, and an optional hint. - // The hint is in the form of a hintfmt class, which wraps boost::format(), and - // makes all the substituted text yellow. - printErrorInfo( - ProgramWarning() - .name("warning name") - .description("warning description") - .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow. - ); + // ProgramWarning takes name, description, and an optional hint. + // The hint is in the form of a hintfmt class, which wraps boost::format(), and + // makes all the substituted text yellow. + printErrorInfo( + ProgramWarning() + .name("warning name") + .description("warning description") + .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow. + ); - /* - // some invalid errors: - - // type error: no hint function. - ProgramError() - .name("name") - .description("error description"); + /* + // some invalid errors: + + // type error: no hint function. + ProgramError() + .name("name") + .description("error description"); - // type error: description before name. - ProgramError() - .description("error description") - .name("name") - .nohint(); + // type error: description before name. + ProgramError() + .description("error description") + .name("name") + .nohint(); - // type error: hint function with regular boost format, not special hintfmt. - ProgramError() - .description("error description") - .name("name") - .hint(format("there was a %1%") % "warning"); - */ + // type error: hint function with regular boost format, not special hintfmt. + ProgramError() + .description("error description") + .name("name") + .hint(format("there was a %1%") % "warning"); + */ - // NixLangWarning adds nix file, line number, column range, and the lines of code - // where a warning occurred. - printErrorInfo( - NixLangWarning() - .name("warning name") - .description("warning description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(nullopt - ,"this is the problem line of code" - ,nullopt) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangWarning adds nix file, line number, column range, and the lines of code + // where a warning occurred. + printErrorInfo( + NixLangWarning() + .name("warning name") + .description("warning description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(std::nullopt + ,"this is the problem line of code" + ,std::nullopt) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); - // NixLangError is just the same as NixLangWarning, except for the Error flag. - printErrorInfo( - NixLangError() - .name("error name") - .description("error description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(optional("previous line of code") - ,"this is the problem line of code" - ,optional("next line of code")) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangError is just the same as NixLangWarning, except for the Error flag. + printErrorInfo( + NixLangError() + .name("error name") + .description("error description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13,7) + .linesOfCode(optional("previous line of code") + ,"this is the problem line of code" + ,optional("next line of code")) + .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") + ); - return 0; + return 0; } From dd7b8183a5065bc0279d9bd87c27dd0ac91ccd8a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 1 Apr 2020 16:20:20 -0600 Subject: [PATCH 069/350] indenting --- src/libutil/error.cc | 256 +++++++++++++++++++++---------------------- src/libutil/error.hh | 244 ++++++++++++++++++++--------------------- 2 files changed, 250 insertions(+), 250 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 2b7a079dc..4480e80bf 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,155 +10,155 @@ optional ErrorInfo::programName = std::nullopt; // return basic_format? string showErrLine(ErrLine &errLine) { - if (errLine.columnRange.has_value()) - { - return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); - } - else - { - return (format("(%1%)") % errLine.lineNumber).str(); - }; + if (errLine.columnRange.has_value()) + { + return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); + } + else + { + return (format("(%1%)") % errLine.lineNumber).str(); + }; } void printCodeLines(string &prefix, NixCode &nixCode) { - + if (nixCode.errLine.has_value()) - { - // previous line of code. - if (nixCode.errLine->prevLineOfCode.has_value()) { - cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber - 1) - % *nixCode.errLine->prevLineOfCode - << endl; - } - - // line of code containing the error.%2$+5d% - cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber) - % nixCode.errLine->errLineOfCode - << endl; - - // error arrows for the column range. - if (nixCode.errLine->columnRange.has_value()) { - int start = nixCode.errLine->columnRange->start; - std::string spaces; - for (int i = 0; i < start; ++i) - { - spaces.append(" "); - } + // previous line of code. + if (nixCode.errLine->prevLineOfCode.has_value()) { + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nixCode.errLine->lineNumber - 1) + % *nixCode.errLine->prevLineOfCode + << endl; + } + + // line of code containing the error.%2$+5d% + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nixCode.errLine->lineNumber) + % nixCode.errLine->errLineOfCode + << endl; + + // error arrows for the column range. + if (nixCode.errLine->columnRange.has_value()) + { + int start = nixCode.errLine->columnRange->start; + std::string spaces; + for (int i = 0; i < start; ++i) + { + spaces.append(" "); + } + + int len = nixCode.errLine->columnRange->len; + std::string arrows; + for (int i = 0; i < len; ++i) + { + arrows.append("^"); + } + + cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl; + } + + + + // next line of code. + if (nixCode.errLine->nextLineOfCode.has_value()) { + cout << format("%1% %|2$5d|| %3%") + % prefix + % (nixCode.errLine->lineNumber + 1) + % *nixCode.errLine->nextLineOfCode + << endl; + } - int len = nixCode.errLine->columnRange->len; - std::string arrows; - for (int i = 0; i < len; ++i) - { - arrows.append("^"); - } - - cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl; } - - - // next line of code. - if (nixCode.errLine->nextLineOfCode.has_value()) { - cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber + 1) - % *nixCode.errLine->nextLineOfCode - << endl; - } - - } - } void printErrorInfo(ErrorInfo &einfo) { - int errwidth = 80; - string prefix = " "; + int errwidth = 80; + string prefix = " "; - string levelString; - switch (einfo.level) - { - case ErrLevel::elError: - { - levelString = ANSI_RED; - levelString += "error:"; - levelString += ANSI_NORMAL; - break; - } - case ErrLevel::elWarning: - { - levelString = ANSI_YELLOW; - levelString += "warning:"; - levelString += ANSI_NORMAL; - break; - } - default: - { - levelString = (format("invalid error level: %1%") % einfo.level).str(); - break; - } - } - - int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); - int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; - - string dashes; - for (int i = 0; i < dashwidth; ++i) - dashes.append("-"); - - // divider. - cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) - % prefix - % levelString - % "---" - % einfo.name - % dashes - % einfo.programName.value_or("") - << endl; - - // filename. - if (einfo.nixCode.has_value()) - { - if (einfo.nixCode->nixFile.has_value()) + string levelString; + switch (einfo.level) { - string eline = einfo.nixCode->errLine.has_value() - ? string(" ") + showErrLine(*einfo.nixCode->errLine) - : ""; - - cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % *einfo.nixCode->nixFile % eline << endl; - cout << prefix << endl; + case ErrLevel::elError: + { + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; + break; + } + case ErrLevel::elWarning: + { + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; + break; + } + default: + { + levelString = (format("invalid error level: %1%") % einfo.level).str(); + break; + } } - else + + int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; + + string dashes; + for (int i = 0; i < dashwidth; ++i) + dashes.append("-"); + + // divider. + cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) + % prefix + % levelString + % "---" + % einfo.name + % dashes + % einfo.programName.value_or("") + << endl; + + // filename. + if (einfo.nixCode.has_value()) { - cout << format("%1%from command line argument") % prefix << endl; - cout << prefix << endl; + if (einfo.nixCode->nixFile.has_value()) + { + string eline = einfo.nixCode->errLine.has_value() + ? string(" ") + showErrLine(*einfo.nixCode->errLine) + : ""; + + cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) + % prefix % *einfo.nixCode->nixFile % eline << endl; + cout << prefix << endl; + } + else + { + cout << format("%1%from command line argument") % prefix << endl; + cout << prefix << endl; + } } - } - // description - cout << prefix << einfo.description << endl; - cout << prefix << endl; - - // lines of code. - if (einfo.nixCode.has_value()) - { - printCodeLines(prefix, *einfo.nixCode); + // description + cout << prefix << einfo.description << endl; cout << prefix << endl; - } - // hint - if (einfo.hint.has_value()) - { - cout << prefix << *einfo.hint << endl; - cout << prefix << endl; - } + // lines of code. + if (einfo.nixCode.has_value()) + { + printCodeLines(prefix, *einfo.nixCode); + cout << prefix << endl; + } + + // hint + if (einfo.hint.has_value()) + { + cout << prefix << *einfo.hint << endl; + cout << prefix << endl; + } } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 614f08a35..00c32ff21 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -11,45 +11,45 @@ namespace nix { -typedef enum - { elWarning - , elError - } ErrLevel; +typedef enum { + elWarning + , elError + } ErrLevel; class ColumnRange { - public: - unsigned int start; - unsigned int len; + public: + unsigned int start; + unsigned int len; }; class ErrorInfo; class ErrLine { - public: - int lineNumber; - optional columnRange; - optional prevLineOfCode; - string errLineOfCode; - optional nextLineOfCode; + public: + int lineNumber; + optional columnRange; + optional prevLineOfCode; + string errLineOfCode; + optional nextLineOfCode; }; class NixCode { - public: - optional nixFile; - optional errLine; + public: + optional nixFile; + optional errLine; - ErrLine& ensureErrLine() - { - if (!this->errLine.has_value()) - this->errLine = optional(ErrLine()); - return *this->errLine; - } + ErrLine& ensureErrLine() + { + if (!this->errLine.has_value()) + this->errLine = optional(ErrLine()); + return *this->errLine; + } }; // ------------------------------------------------- // ErrorInfo. -// Forward friend class declarations. "builder classes" +// Forward friend class declarations. "builder classes" template class AddName; @@ -76,170 +76,170 @@ class AddLOC; // The error info class itself. class ErrorInfo { - public: - ErrLevel level; - string name; - string description; - optional nixCode; - optional hint; - ErrorInfo& GetEI() { return *this; } + public: + ErrLevel level; + string name; + string description; + optional nixCode; + optional hint; + ErrorInfo& GetEI() { return *this; } - static optional programName; + static optional programName; - // give these access to the private constructor, - // when they are direct descendants (children but not grandchildren). - friend AddName; - friend AddDescription; - friend AddNixCode; - friend AddNixFile; - friend AddErrLine; - friend AddLineNumber; - friend AddColumnRange; - friend AddLOC; - - NixCode& ensureNixCode() - { - if (!this->nixCode.has_value()) - this->nixCode = optional(NixCode()); - return *this->nixCode; - } - protected: - // constructor is protected, so only the builder classes can create an ErrorInfo. - ErrorInfo(ErrLevel level) { this->level = level; } + // give these access to the private constructor, + // when they are direct descendants (children but not grandchildren). + friend AddName; + friend AddDescription; + friend AddNixCode; + friend AddNixFile; + friend AddErrLine; + friend AddLineNumber; + friend AddColumnRange; + friend AddLOC; + + NixCode& ensureNixCode() + { + if (!this->nixCode.has_value()) + this->nixCode = optional(NixCode()); + return *this->nixCode; + } + protected: + // constructor is protected, so only the builder classes can create an ErrorInfo. + ErrorInfo(ErrLevel level) { this->level = level; } }; // Init as error class EIError : public ErrorInfo { - protected: - EIError() : ErrorInfo(elError) {} + protected: + EIError() : ErrorInfo(elError) {} }; // Init as warning class EIWarning : public ErrorInfo { - protected: - EIWarning() : ErrorInfo(elWarning) {} + protected: + EIWarning() : ErrorInfo(elWarning) {} }; // Builder class definitions. template class AddName : private T { - public: - T& name(const std::string &name){ - GetEI().name = name; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& name(const std::string &name){ + GetEI().name = name; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class AddDescription : private T { - public: - T& description(const std::string &description){ - GetEI().description = description; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& description(const std::string &description){ + GetEI().description = description; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class AddNixFile : private T { - public: - T& nixFile(string filename) { - GetEI().ensureNixCode().nixFile = filename; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& nixFile(string filename) { + GetEI().ensureNixCode().nixFile = filename; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class AddLineNumber : private T { - public: - T& lineNumber(int lineNumber) { - GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& lineNumber(int lineNumber) { + GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class AddColumnRange : private T { - public: - T& columnRange(unsigned int start, unsigned int len) { - GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& columnRange(unsigned int start, unsigned int len) { + GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class AddLOC : private T { - public: - T& linesOfCode(optional prevloc, string loc, optional nextloc) { - GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; - GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; - GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& linesOfCode(optional prevloc, string loc, optional nextloc) { + GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; + GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; + GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; template class yellowify { public: - yellowify(T &s) : value(s) {} - T &value; + yellowify(T &s) : value(s) {} + T &value; }; template std::ostream& operator<<(std::ostream &out, const yellowify &y) { - return out << ANSI_YELLOW << y.value << ANSI_NORMAL; + return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } // hint format shows templated values in yellow. class hintfmt { - public: - hintfmt(string format) :fmt(format) {} - template - hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; } + public: + hintfmt(string format) :fmt(format) {} + template + hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; } + + template + friend class AddHint; + private: + format fmt; - template - friend class AddHint; - private: - format fmt; - }; template class AddHint : private T { - public: - T& hint(hintfmt &hfmt) { - GetEI().hint = optional(hfmt.fmt.str()); - return *this; - } - T& nohint() { - GetEI().hint = std::nullopt; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } + public: + T& hint(hintfmt &hfmt) { + GetEI().hint = optional(hfmt.fmt.str()); + return *this; + } + T& nohint() { + GetEI().hint = std::nullopt; + return *this; + } + protected: + ErrorInfo& GetEI() { return T::GetEI(); } }; // -------------------------------------------------------- From e697884f654fb48f0fee05de25910b27fa147f07 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 1 Apr 2020 21:30:19 -0600 Subject: [PATCH 070/350] using std:: everywhere; fix a formatting error; add exception flags --- src/libutil/error.cc | 41 ++++++++++++++++++++--------------------- src/libutil/error.hh | 35 +++++++++++++++++++++-------------- tests/errors/main.cc | 8 +++----- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 4480e80bf..04da075ed 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -5,9 +5,8 @@ namespace nix { -optional ErrorInfo::programName = std::nullopt; +std::optional ErrorInfo::programName = std::nullopt; -// return basic_format? string showErrLine(ErrLine &errLine) { if (errLine.columnRange.has_value()) @@ -27,19 +26,19 @@ void printCodeLines(string &prefix, NixCode &nixCode) { // previous line of code. if (nixCode.errLine->prevLineOfCode.has_value()) { - cout << format("%1% %|2$5d|| %3%") + std::cout << format("%1% %|2$5d|| %3%") % prefix % (nixCode.errLine->lineNumber - 1) % *nixCode.errLine->prevLineOfCode - << endl; + << std::endl; } // line of code containing the error.%2$+5d% - cout << format("%1% %|2$5d|| %3%") + std::cout << format("%1% %|2$5d|| %3%") % prefix % (nixCode.errLine->lineNumber) % nixCode.errLine->errLineOfCode - << endl; + << std::endl; // error arrows for the column range. if (nixCode.errLine->columnRange.has_value()) @@ -58,18 +57,18 @@ void printCodeLines(string &prefix, NixCode &nixCode) arrows.append("^"); } - cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << endl; + std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; } // next line of code. if (nixCode.errLine->nextLineOfCode.has_value()) { - cout << format("%1% %|2$5d|| %3%") + std::cout << format("%1% %|2$5d|| %3%") % prefix % (nixCode.errLine->lineNumber + 1) % *nixCode.errLine->nextLineOfCode - << endl; + << std::endl; } } @@ -113,14 +112,14 @@ void printErrorInfo(ErrorInfo &einfo) dashes.append("-"); // divider. - cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) + std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) % prefix % levelString % "---" % einfo.name % dashes % einfo.programName.value_or("") - << endl; + << std::endl; // filename. if (einfo.nixCode.has_value()) @@ -131,33 +130,33 @@ void printErrorInfo(ErrorInfo &einfo) ? string(" ") + showErrLine(*einfo.nixCode->errLine) : ""; - cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % *einfo.nixCode->nixFile % eline << endl; - cout << prefix << endl; + std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) + % prefix % *einfo.nixCode->nixFile % eline << std::endl; + std::cout << prefix << std::endl; } else { - cout << format("%1%from command line argument") % prefix << endl; - cout << prefix << endl; + std::cout << format("%1%from command line argument") % prefix << std::endl; + std::cout << prefix << std::endl; } } // description - cout << prefix << einfo.description << endl; - cout << prefix << endl; + std::cout << prefix << einfo.description << std::endl; + std::cout << prefix << std::endl; // lines of code. if (einfo.nixCode.has_value()) { printCodeLines(prefix, *einfo.nixCode); - cout << prefix << endl; + std::cout << prefix << std::endl; } // hint if (einfo.hint.has_value()) { - cout << prefix << *einfo.hint << endl; - cout << prefix << endl; + std::cout << prefix << *einfo.hint << std::endl; + std::cout << prefix << std::endl; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 00c32ff21..9dcb42e14 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -27,21 +27,21 @@ class ErrorInfo; class ErrLine { public: int lineNumber; - optional columnRange; - optional prevLineOfCode; + std::optional columnRange; + std::optional prevLineOfCode; string errLineOfCode; - optional nextLineOfCode; + std::optional nextLineOfCode; }; class NixCode { public: - optional nixFile; - optional errLine; + std::optional nixFile; + std::optional errLine; ErrLine& ensureErrLine() { if (!this->errLine.has_value()) - this->errLine = optional(ErrLine()); + this->errLine = std::optional(ErrLine()); return *this->errLine; } }; @@ -80,11 +80,11 @@ class ErrorInfo { ErrLevel level; string name; string description; - optional nixCode; - optional hint; + std::optional nixCode; + std::optional hint; ErrorInfo& GetEI() { return *this; } - static optional programName; + static std::optional programName; // give these access to the private constructor, // when they are direct descendants (children but not grandchildren). @@ -100,7 +100,7 @@ class ErrorInfo { NixCode& ensureNixCode() { if (!this->nixCode.has_value()) - this->nixCode = optional(NixCode()); + this->nixCode = std::optional(NixCode()); return *this->nixCode; } protected: @@ -187,7 +187,7 @@ template class AddLOC : private T { public: - T& linesOfCode(optional prevloc, string loc, optional nextloc) { + T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) { GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; @@ -197,6 +197,11 @@ class AddLOC : private T ErrorInfo& GetEI() { return T::GetEI(); } }; + +// ---------------------------------------------------------------- +// format for hints. same as boost format, except templated values +// are always in yellow. + template class yellowify { @@ -211,11 +216,12 @@ std::ostream& operator<<(std::ostream &out, const yellowify &y) return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } -// hint format shows templated values in yellow. class hintfmt { public: - hintfmt(string format) :fmt(format) {} + hintfmt(string format) :fmt(format) { + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + } template hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; } @@ -226,12 +232,13 @@ class hintfmt }; +// the template layer for adding a hint. template class AddHint : private T { public: T& hint(hintfmt &hfmt) { - GetEI().hint = optional(hfmt.fmt.str()); + GetEI().hint = std::optional(hfmt.fmt.str()); return *this; } T& nohint() { diff --git a/tests/errors/main.cc b/tests/errors/main.cc index f11390c73..870af75d3 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -3,14 +3,12 @@ #include #include - - int main() { using namespace nix; // In each program where errors occur, this has to be set. - ErrorInfo::programName = optional("error-test"); + ErrorInfo::programName = std::optional("error-test"); // There are currently four error types: // @@ -83,9 +81,9 @@ int main() .nixFile("myfile.nix") .lineNumber(40) .columnRange(13,7) - .linesOfCode(optional("previous line of code") + .linesOfCode(std::optional("previous line of code") ,"this is the problem line of code" - ,optional("next line of code")) + ,std::optional("next line of code")) .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") ); From c34e96f7e0d53894f302134b2f90f83b20ffd22a Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 2 Apr 2020 05:03:58 +0200 Subject: [PATCH 071/350] Make function arguments retain position info This allows querying the location of function arguments. E.g. builtins.unsafeGetAttrPos "x" (builtins.functionArgs ({ x }: null)) => { column = 57; file = "/home/infinisil/src/nix/inst/test.nix"; line = 1; } --- src/libexpr/nixexpr.hh | 3 ++- src/libexpr/parser.y | 4 ++-- src/libexpr/primops.cc | 7 +++++-- tests/lang/eval-okay-getattrpos-functionargs.exp | 1 + tests/lang/eval-okay-getattrpos-functionargs.nix | 4 ++++ 5 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.exp create mode 100644 tests/lang/eval-okay-getattrpos-functionargs.nix diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index f7e9105a4..8c96de37c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -209,9 +209,10 @@ struct ExprList : Expr struct Formal { + Pos pos; Symbol name; Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; + Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { }; }; struct Formals diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..08cec96dc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -531,8 +531,8 @@ formals ; formal - : ID { $$ = new Formal(data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); } ; %% diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..81c378dff 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1354,9 +1354,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) + for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i.name), i.def); + Value * value = state.allocValue(); + v.attrs->push_back(Attr(i.name, value, &i.pos)); + mkBool(*value, i.def); + } v.attrs->sort(); } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp new file mode 100644 index 000000000..7f9ac40e8 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.exp @@ -0,0 +1 @@ +{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix new file mode 100644 index 000000000..11d6bb0e3 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.nix @@ -0,0 +1,4 @@ +let + fun = { foo }: {}; + pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun); +in { inherit (pos) column line; file = baseNameOf pos.file; } From 4fc4eb6c93511c61a278ba0e5555ffb24435ab59 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 2 Apr 2020 17:04:00 +0200 Subject: [PATCH 072/350] libexpr: remove unused attrError The attrError variable is no longer used but still allocated on every call to the findAlongAttrPath function. --- src/libexpr/attr-path.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 4545bfd72..76d101b98 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -37,9 +37,6 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr { Strings tokens = parseAttrPath(attrPath); - Error attrError = - Error(format("attribute selection path '%1%' does not match expression") % attrPath); - Value * v = &vIn; Pos pos = noPos; From b85ba3e30da663a00847a30fd9ceb1bc8284bafc Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Apr 2020 14:08:05 -0600 Subject: [PATCH 073/350] full include path --- tests/errors/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 870af75d3..5de36a37d 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -1,4 +1,4 @@ -#include "error.hh" +#include "../../src/libutil/error.hh" #include #include From 1c329ca433e78ba9018a54f5d499c7cabe3c7b66 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Apr 2020 14:25:43 -0600 Subject: [PATCH 074/350] indenting --- src/libutil/error.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 9dcb42e14..32571d97f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -12,9 +12,9 @@ namespace nix { typedef enum { - elWarning - , elError - } ErrLevel; + elWarning, + elError +} ErrLevel; class ColumnRange { public: From c6b3fcddb0d05718ec11807ad847d7bf60cc703f Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 2 Apr 2020 16:02:40 -0600 Subject: [PATCH 075/350] formatted with astyle --- src/libutil/error.cc | 136 ++++++++--------- src/libutil/error.hh | 351 ++++++++++++++++++++++++------------------- tests/errors/main.cc | 103 ++++++------- 3 files changed, 308 insertions(+), 282 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 04da075ed..8ab4a2dea 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -3,109 +3,99 @@ #include #include -namespace nix { +namespace nix +{ std::optional ErrorInfo::programName = std::nullopt; string showErrLine(ErrLine &errLine) { - if (errLine.columnRange.has_value()) - { + if (errLine.columnRange.has_value()) { return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); - } - else - { + } else { return (format("(%1%)") % errLine.lineNumber).str(); }; } -void printCodeLines(string &prefix, NixCode &nixCode) +void printCodeLines(string &prefix, NixCode &nixCode) { - if (nixCode.errLine.has_value()) - { + if (nixCode.errLine.has_value()) { // previous line of code. - if (nixCode.errLine->prevLineOfCode.has_value()) { + if (nixCode.errLine->prevLineOfCode.has_value()) { std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber - 1) - % *nixCode.errLine->prevLineOfCode - << std::endl; + % prefix + % (nixCode.errLine->lineNumber - 1) + % *nixCode.errLine->prevLineOfCode + << std::endl; } // line of code containing the error.%2$+5d% std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber) - % nixCode.errLine->errLineOfCode - << std::endl; + % prefix + % (nixCode.errLine->lineNumber) + % nixCode.errLine->errLineOfCode + << std::endl; - // error arrows for the column range. - if (nixCode.errLine->columnRange.has_value()) - { + // error arrows for the column range. + if (nixCode.errLine->columnRange.has_value()) { int start = nixCode.errLine->columnRange->start; std::string spaces; - for (int i = 0; i < start; ++i) - { + for (int i = 0; i < start; ++i) { spaces.append(" "); } int len = nixCode.errLine->columnRange->len; std::string arrows; - for (int i = 0; i < len; ++i) - { + for (int i = 0; i < len; ++i) { arrows.append("^"); } - std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; + std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; } // next line of code. - if (nixCode.errLine->nextLineOfCode.has_value()) { + if (nixCode.errLine->nextLineOfCode.has_value()) { std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber + 1) - % *nixCode.errLine->nextLineOfCode - << std::endl; + % prefix + % (nixCode.errLine->lineNumber + 1) + % *nixCode.errLine->nextLineOfCode + << std::endl; } } } -void printErrorInfo(ErrorInfo &einfo) +void printErrorInfo(ErrorInfo &einfo) { int errwidth = 80; string prefix = " "; string levelString; - switch (einfo.level) - { - case ErrLevel::elError: - { - levelString = ANSI_RED; - levelString += "error:"; - levelString += ANSI_NORMAL; - break; - } - case ErrLevel::elWarning: - { - levelString = ANSI_YELLOW; - levelString += "warning:"; - levelString += ANSI_NORMAL; - break; - } - default: - { - levelString = (format("invalid error level: %1%") % einfo.level).str(); - break; - } + switch (einfo.level) { + case ErrLevel::elError: { + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; + break; + } + case ErrLevel::elWarning: { + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; + break; + } + default: { + levelString = (format("invalid error level: %1%") % einfo.level).str(); + break; + } } int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); - int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; + int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; string dashes; for (int i = 0; i < dashwidth; ++i) @@ -113,29 +103,25 @@ void printErrorInfo(ErrorInfo &einfo) // divider. std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) - % prefix - % levelString - % "---" - % einfo.name - % dashes - % einfo.programName.value_or("") - << std::endl; + % prefix + % levelString + % "---" + % einfo.name + % dashes + % einfo.programName.value_or("") + << std::endl; // filename. - if (einfo.nixCode.has_value()) - { - if (einfo.nixCode->nixFile.has_value()) - { + if (einfo.nixCode.has_value()) { + if (einfo.nixCode->nixFile.has_value()) { string eline = einfo.nixCode->errLine.has_value() - ? string(" ") + showErrLine(*einfo.nixCode->errLine) - : ""; + ? string(" ") + showErrLine(*einfo.nixCode->errLine) + : ""; - std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % *einfo.nixCode->nixFile % eline << std::endl; + std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) + % prefix % *einfo.nixCode->nixFile % eline << std::endl; std::cout << prefix << std::endl; - } - else - { + } else { std::cout << format("%1%from command line argument") % prefix << std::endl; std::cout << prefix << std::endl; } @@ -146,15 +132,13 @@ void printErrorInfo(ErrorInfo &einfo) std::cout << prefix << std::endl; // lines of code. - if (einfo.nixCode.has_value()) - { + if (einfo.nixCode.has_value()) { printCodeLines(prefix, *einfo.nixCode); std::cout << prefix << std::endl; } // hint - if (einfo.hint.has_value()) - { + if (einfo.hint.has_value()) { std::cout << prefix << *einfo.hint << std::endl; std::cout << prefix << std::endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 32571d97f..70884c189 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -9,41 +9,45 @@ #include -namespace nix { +namespace nix +{ -typedef enum { +typedef enum { elWarning, elError } ErrLevel; -class ColumnRange { - public: - unsigned int start; - unsigned int len; +class ColumnRange +{ +public: + unsigned int start; + unsigned int len; }; class ErrorInfo; -class ErrLine { - public: - int lineNumber; - std::optional columnRange; - std::optional prevLineOfCode; - string errLineOfCode; - std::optional nextLineOfCode; +class ErrLine +{ +public: + int lineNumber; + std::optional columnRange; + std::optional prevLineOfCode; + string errLineOfCode; + std::optional nextLineOfCode; }; -class NixCode { - public: - std::optional nixFile; - std::optional errLine; +class NixCode +{ +public: + std::optional nixFile; + std::optional errLine; - ErrLine& ensureErrLine() - { - if (!this->errLine.has_value()) - this->errLine = std::optional(ErrLine()); - return *this->errLine; - } + ErrLine& ensureErrLine() + { + if (!this->errLine.has_value()) + this->errLine = std::optional(ErrLine()); + return *this->errLine; + } }; // ------------------------------------------------- @@ -65,149 +69,180 @@ class AddNixFile; template class AddErrLine; -template +template class AddLineNumber; -template +template class AddColumnRange; -template +template class AddLOC; // The error info class itself. -class ErrorInfo { - public: - ErrLevel level; - string name; - string description; - std::optional nixCode; - std::optional hint; - ErrorInfo& GetEI() { return *this; } +class ErrorInfo +{ +public: + ErrLevel level; + string name; + string description; + std::optional nixCode; + std::optional hint; + ErrorInfo& GetEI() + { + return *this; + } - static std::optional programName; + static std::optional programName; - // give these access to the private constructor, - // when they are direct descendants (children but not grandchildren). - friend AddName; - friend AddDescription; - friend AddNixCode; - friend AddNixFile; - friend AddErrLine; - friend AddLineNumber; - friend AddColumnRange; - friend AddLOC; + // give these access to the private constructor, + // when they are direct descendants (children but not grandchildren). + friend AddName; + friend AddDescription; + friend AddNixCode; + friend AddNixFile; + friend AddErrLine; + friend AddLineNumber; + friend AddColumnRange; + friend AddLOC; - NixCode& ensureNixCode() - { - if (!this->nixCode.has_value()) - this->nixCode = std::optional(NixCode()); - return *this->nixCode; - } - protected: - // constructor is protected, so only the builder classes can create an ErrorInfo. - ErrorInfo(ErrLevel level) { this->level = level; } + NixCode& ensureNixCode() + { + if (!this->nixCode.has_value()) + this->nixCode = std::optional(NixCode()); + return *this->nixCode; + } +protected: + // constructor is protected, so only the builder classes can create an ErrorInfo. + ErrorInfo(ErrLevel level) + { + this->level = level; + } }; // Init as error class EIError : public ErrorInfo { - protected: - EIError() : ErrorInfo(elError) {} +protected: + EIError() : ErrorInfo(elError) {} }; // Init as warning class EIWarning : public ErrorInfo { - protected: - EIWarning() : ErrorInfo(elWarning) {} +protected: + EIWarning() : ErrorInfo(elWarning) {} }; // Builder class definitions. template class AddName : private T { - public: - T& name(const std::string &name){ - GetEI().name = name; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& name(const std::string &name) + { + GetEI().name = name; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; template -class AddDescription : private T +class AddDescription : private T { - public: - T& description(const std::string &description){ - GetEI().description = description; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& description(const std::string &description) + { + GetEI().description = description; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; -template +template class AddNixFile : private T { - public: - T& nixFile(string filename) { - GetEI().ensureNixCode().nixFile = filename; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& nixFile(string filename) + { + GetEI().ensureNixCode().nixFile = filename; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; -template +template class AddLineNumber : private T { - public: - T& lineNumber(int lineNumber) { - GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& lineNumber(int lineNumber) + { + GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; -template +template class AddColumnRange : private T { - public: - T& columnRange(unsigned int start, unsigned int len) { - GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& columnRange(unsigned int start, unsigned int len) + { + GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; -template +template class AddLOC : private T { - public: - T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) { - GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; - GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; - GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) + { + GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; + GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; + GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; // ---------------------------------------------------------------- -// format for hints. same as boost format, except templated values +// format for hints. same as boost format, except templated values // are always in yellow. template class yellowify { public: - yellowify(T &s) : value(s) {} - T &value; + yellowify(T &s) : value(s) {} + T &value; }; template @@ -216,69 +251,79 @@ std::ostream& operator<<(std::ostream &out, const yellowify &y) return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } -class hintfmt +class hintfmt { - public: - hintfmt(string format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - } - template - hintfmt& operator%(const T &value) { fmt % yellowify(value); return *this; } +public: + hintfmt(string format) :fmt(format) + { + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + } + template + hintfmt& operator%(const T &value) + { + fmt % yellowify(value); + return *this; + } - template - friend class AddHint; - private: - format fmt; + template + friend class AddHint; +private: + format fmt; }; // the template layer for adding a hint. -template +template class AddHint : private T { - public: - T& hint(hintfmt &hfmt) { - GetEI().hint = std::optional(hfmt.fmt.str()); - return *this; - } - T& nohint() { - GetEI().hint = std::nullopt; - return *this; - } - protected: - ErrorInfo& GetEI() { return T::GetEI(); } +public: + T& hint(hintfmt &hfmt) + { + GetEI().hint = std::optional(hfmt.fmt.str()); + return *this; + } + T& nohint() + { + GetEI().hint = std::nullopt; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } }; // -------------------------------------------------------- // error types typedef AddName< - AddDescription< - AddHint< - EIError>>> ProgramError; +AddDescription< +AddHint< +EIError>>> ProgramError; typedef AddName< - AddDescription< - AddHint< - EIWarning>>> ProgramWarning; +AddDescription< +AddHint< +EIWarning>>> ProgramWarning; typedef AddName< - AddDescription< - AddNixFile< - AddLineNumber< - AddColumnRange< - AddLOC< - AddHint< - EIError>>>>>>> NixLangError; +AddDescription< +AddNixFile< +AddLineNumber< +AddColumnRange< +AddLOC< +AddHint< +EIError>>>>>>> NixLangError; typedef AddName< - AddDescription< - AddNixFile< - AddLineNumber< - AddColumnRange< - AddLOC< - AddHint< - EIWarning>>>>>>> NixLangWarning; +AddDescription< +AddNixFile< +AddLineNumber< +AddColumnRange< +AddLOC< +AddHint< +EIWarning>>>>>>> NixLangWarning; // -------------------------------------------------------- diff --git a/tests/errors/main.cc b/tests/errors/main.cc index 5de36a37d..e035882e7 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -1,45 +1,45 @@ #include "../../src/libutil/error.hh" -#include #include +#include -int main() +int main() { - using namespace nix; + using namespace nix; // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-test"); // There are currently four error types: - // + // // ProgramError, ProgramWarning, NixLangError, NixLangWarning. - // + // // Each error type is created with a specific sequence of builder functions. // Unlike with a constructor, each parameter is clearly named. - // If the sequence of function calls isn't followed, then there's a type error. - // This should make for a consistent look in the code when errors are created. + // If the sequence of function calls isn't followed, then there's a type + // error. This should make for a consistent look in the code when errors are + // created. // ProgramError takes name, description, and an optional hint. - printErrorInfo( - ProgramError() - .name("name") - .description("error description") - .nohint() - ); + printErrorInfo( ProgramError() + .name("name") + .description("error description") + .nohint() + ); // ProgramWarning takes name, description, and an optional hint. - // The hint is in the form of a hintfmt class, which wraps boost::format(), and - // makes all the substituted text yellow. - printErrorInfo( - ProgramWarning() - .name("warning name") - .description("warning description") - .hint(hintfmt("there was a %1%") % "warning") // 'warning' will be yellow. - ); + // The hint is in the form of a hintfmt class, which wraps boost::format(), + // and makes all the substituted text yellow. + printErrorInfo( ProgramWarning() + .name("warning name") + .description("warning description") + .hint(hintfmt("there was a %1%") % + "warning") // 'warning' will be yellow. + ); /* // some invalid errors: - + // type error: no hint function. ProgramError() .name("name") @@ -51,41 +51,38 @@ int main() .name("name") .nohint(); - // type error: hint function with regular boost format, not special hintfmt. - ProgramError() - .description("error description") - .name("name") + // type error: hint function with regular boost format, not special + hintfmt. ProgramError() .description("error description") .name("name") .hint(format("there was a %1%") % "warning"); */ - // NixLangWarning adds nix file, line number, column range, and the lines of code - // where a warning occurred. - printErrorInfo( - NixLangWarning() - .name("warning name") - .description("warning description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(std::nullopt - ,"this is the problem line of code" - ,std::nullopt) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangWarning adds nix file, line number, column range, and the lines of + // code where a warning occurred. + printErrorInfo(NixLangWarning() + .name("warning name") + .description("warning description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13, 7) + .linesOfCode(std::nullopt, + "this is the problem line of code", + std::nullopt) + .hint(hintfmt("this hint has %1% templated %2%!!") % + "yellow" % "values")); - // NixLangError is just the same as NixLangWarning, except for the Error flag. - printErrorInfo( - NixLangError() - .name("error name") - .description("error description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13,7) - .linesOfCode(std::optional("previous line of code") - ,"this is the problem line of code" - ,std::optional("next line of code")) - .hint(hintfmt("this hint has %1% templated %2%!!") % "yellow" % "values") - ); + // NixLangError is just the same as NixLangWarning, except for the Error + // flag. + printErrorInfo(NixLangError() + .name("error name") + .description("error description") + .nixFile("myfile.nix") + .lineNumber(40) + .columnRange(13, 7) + .linesOfCode(std::optional("previous line of code"), + "this is the problem line of code", + std::optional("next line of code")) + .hint(hintfmt("this hint has %1% templated %2%!!") % + "yellow" % "values")); return 0; } From 7b7801d3f0e0e1cd32b1279979970ad71b66b879 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 3 Apr 2020 08:48:20 -0600 Subject: [PATCH 076/350] variadic args for hint format --- src/libutil/error.hh | 26 ++++++++++++++++++++------ src/libutil/types.hh | 10 ++++++---- tests/errors/main.cc | 10 ++++------ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 70884c189..1c5d6d13c 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -7,6 +7,7 @@ #include #include +#include "types.hh" #include namespace nix @@ -234,7 +235,7 @@ protected: // ---------------------------------------------------------------- -// format for hints. same as boost format, except templated values +// format for hints. same as fmt, except templated values // are always in yellow. template @@ -251,20 +252,25 @@ std::ostream& operator<<(std::ostream &out, const yellowify &y) return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } -class hintfmt +class hintformat { public: - hintfmt(string format) :fmt(format) + hintformat(string format) :fmt(format) { fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); } template - hintfmt& operator%(const T &value) + hintformat& operator%(const T &value) { fmt % yellowify(value); return *this; } + std::string str() const + { + return fmt.str(); + } + template friend class AddHint; private: @@ -272,14 +278,22 @@ private: }; +template +inline hintformat hintfmt(const std::string & fs, const Args & ... args) +{ + hintformat f(fs); + formatHelper(f, args...); + return f; +} + // the template layer for adding a hint. template class AddHint : private T { public: - T& hint(hintfmt &hfmt) + T& hint(const hintformat &hf) { - GetEI().hint = std::optional(hfmt.fmt.str()); + GetEI().hint = std::optional(hf.str()); return *this; } T& nohint() diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 20b96a85c..981af528b 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -41,7 +41,8 @@ struct FormatOrString { string s; FormatOrString(const string & s) : s(s) { }; - FormatOrString(const format & f) : s(f.str()) { }; + template + FormatOrString(const F & f) : s(f.str()) { }; FormatOrString(const char * s) : s(s) { }; }; @@ -51,12 +52,13 @@ struct FormatOrString ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion takes place). */ -inline void formatHelper(boost::format & f) +template +inline void formatHelper(F & f) { } -template -inline void formatHelper(boost::format & f, const T & x, const Args & ... args) +template +inline void formatHelper(F & f, const T & x, const Args & ... args) { formatHelper(f % x, args...); } diff --git a/tests/errors/main.cc b/tests/errors/main.cc index e035882e7..be98bd5b8 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -33,8 +33,8 @@ int main() printErrorInfo( ProgramWarning() .name("warning name") .description("warning description") - .hint(hintfmt("there was a %1%") % - "warning") // 'warning' will be yellow. + // the templated value, 'warning', is automatically colored yellow. + .hint(hintfmt("there was a %1%", "warning")) ); /* @@ -67,8 +67,7 @@ int main() .linesOfCode(std::nullopt, "this is the problem line of code", std::nullopt) - .hint(hintfmt("this hint has %1% templated %2%!!") % - "yellow" % "values")); + .hint(hintfmt("this hint has %1% templated %2%!!", "yellow" , "values"))); // NixLangError is just the same as NixLangWarning, except for the Error // flag. @@ -81,8 +80,7 @@ int main() .linesOfCode(std::optional("previous line of code"), "this is the problem line of code", std::optional("next line of code")) - .hint(hintfmt("this hint has %1% templated %2%!!") % - "yellow" % "values")); + .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); return 0; } From 63fa92605b2b0a8a6d948a27e1d331cc46b199bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 11:54:48 +0200 Subject: [PATCH 077/350] nix-env: Refuse to operate on a new-style profile This prevents users from accidentally nuking their profile via nix-env. (cherry picked from commit 021634e3e3edb327089d33ab41b743f0a40126da) --- src/nix-env/user-env.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 717431b7a..f852916d8 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -15,6 +15,8 @@ namespace nix { DrvInfos queryInstalled(EvalState & state, const Path & userEnv) { DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); Path manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; From 5e7ccdc9e3ddd61dc85e20c898001345bfb497a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Apr 2020 20:06:26 +0200 Subject: [PATCH 078/350] Publish a tarball containing the crates we depend on This is needed since we no longer produce a source tarball. (cherry picked from commit bf70a047a0b2da606f65a88f13ce2994065cd9c4) --- release.nix | 94 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 41 deletions(-) diff --git a/release.nix b/release.nix index 9b591229a..65b035957 100644 --- a/release.nix +++ b/release.nix @@ -12,52 +12,64 @@ let builtins.readFile ./.version + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); + # Create a "vendor" directory that contains the crates listed in + # Cargo.lock. This allows Nix to be built without network access. + vendoredCrates' = + let + lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); + + files = map (pkg: import { + url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; + sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; + }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); + + in pkgs.runCommand "cargo-vendor-dir" {} + '' + mkdir -p $out/vendor + + cat > $out/vendor/config < "$dir/.cargo-checksum.json" + + # Clean up some cruft from the winapi crates. FIXME: find + # a way to remove winapi* from our dependencies. + if [[ $dir =~ /winapi ]]; then + find $dir -name "*.a" -print0 | xargs -0 rm -f -- + fi + + mv "$dir" $out/vendor/ + + rm -rf $out/vendor/tmp + '') files)} + ''; + jobs = rec { - # Create a "vendor" directory that contains the crates listed in - # Cargo.lock. This allows Nix to be built without network access. vendoredCrates = - let - lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); - - files = map (pkg: import { - url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; - sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; - }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); - - in pkgs.runCommand "cargo-vendor-dir" {} + with pkgs; + runCommand "vendored-crates" {} '' - mkdir -p $out/vendor - - cat > $out/vendor/config < "$dir/.cargo-checksum.json" - - # Clean up some cruft from the winapi crates. FIXME: find - # a way to remove winapi* from our dependencies. - if [[ $dir =~ /winapi ]]; then - find $dir -name "*.a" -print0 | xargs -0 rm -f -- - fi - - mv "$dir" $out/vendor/ - - rm -rf $out/vendor/tmp - '') files)} + mkdir -p $out/nix-support + name=nix-vendored-crates-${version} + fn=$out/$name.tar.xz + tar cvfJ $fn -C ${vendoredCrates'} vendor \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --transform "s,vendor,$name," + echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products ''; - build = pkgs.lib.genAttrs systems (system: let pkgs = import nixpkgs { inherit system; }; in @@ -89,7 +101,7 @@ let patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} - ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor (cd perl; autoreconf --install --force --verbose) ''; From 9bb528d3920559454a1ce0f8f4ac3ba58b18f6d1 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 3 Apr 2020 13:15:59 -0600 Subject: [PATCH 079/350] handle Pos instead of individual file/line/columnrange args --- src/libutil/error.hh | 98 +++++++++++--------------------------------- tests/errors/main.cc | 13 +++--- 2 files changed, 31 insertions(+), 80 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1c5d6d13c..57a9944be 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -8,7 +8,6 @@ #include #include "types.hh" -#include namespace nix { @@ -62,19 +61,7 @@ template class AddDescription; template -class AddNixCode; - -template -class AddNixFile; - -template -class AddErrLine; - -template -class AddLineNumber; - -template -class AddColumnRange; +class AddPos; template class AddLOC; @@ -99,11 +86,7 @@ public: // when they are direct descendants (children but not grandchildren). friend AddName; friend AddDescription; - friend AddNixCode; - friend AddNixFile; - friend AddErrLine; - friend AddLineNumber; - friend AddColumnRange; + friend AddPos; friend AddLOC; NixCode& ensureNixCode() @@ -168,44 +151,15 @@ protected: }; template -class AddNixFile : private T +class AddPos : private T { public: - T& nixFile(string filename) + template + T& pos(const P &aPos) { - GetEI().ensureNixCode().nixFile = filename; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddLineNumber : private T -{ -public: - T& lineNumber(int lineNumber) - { - GetEI().ensureNixCode().ensureErrLine().lineNumber = lineNumber; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddColumnRange : private T -{ -public: - T& columnRange(unsigned int start, unsigned int len) - { - GetEI().ensureNixCode().ensureErrLine().columnRange = { start, len }; + GetEI().ensureNixCode().nixFile = aPos.file; + GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line; + GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 }; return *this; } protected: @@ -312,32 +266,28 @@ protected: // error types typedef AddName< -AddDescription< -AddHint< -EIError>>> ProgramError; + AddDescription< + AddHint< + EIError>>> ProgramError; typedef AddName< -AddDescription< -AddHint< -EIWarning>>> ProgramWarning; + AddDescription< + AddHint< + EIWarning>>> ProgramWarning; typedef AddName< -AddDescription< -AddNixFile< -AddLineNumber< -AddColumnRange< -AddLOC< -AddHint< -EIError>>>>>>> NixLangError; + AddDescription< + AddPos< + AddLOC< + AddHint< + EIError>>>>> NixLangError; typedef AddName< -AddDescription< -AddNixFile< -AddLineNumber< -AddColumnRange< -AddLOC< -AddHint< -EIWarning>>>>>>> NixLangWarning; + AddDescription< + AddPos< + AddLOC< + AddHint< + EIWarning>>>>> NixLangWarning; // -------------------------------------------------------- diff --git a/tests/errors/main.cc b/tests/errors/main.cc index be98bd5b8..cd6ca7caf 100644 --- a/tests/errors/main.cc +++ b/tests/errors/main.cc @@ -1,4 +1,5 @@ #include "../../src/libutil/error.hh" +#include "../../src/libexpr/nixexpr.hh" #include #include @@ -58,12 +59,14 @@ int main() // NixLangWarning adds nix file, line number, column range, and the lines of // code where a warning occurred. + + SymbolTable testTable; + auto problem_symbol = testTable.create("problem"); + printErrorInfo(NixLangWarning() .name("warning name") .description("warning description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13, 7) + .pos(Pos(problem_symbol, 40, 13)) .linesOfCode(std::nullopt, "this is the problem line of code", std::nullopt) @@ -74,9 +77,7 @@ int main() printErrorInfo(NixLangError() .name("error name") .description("error description") - .nixFile("myfile.nix") - .lineNumber(40) - .columnRange(13, 7) + .pos(Pos(problem_symbol, 40, 13)) .linesOfCode(std::optional("previous line of code"), "this is the problem line of code", std::optional("next line of code")) From 9a8b3e97477f1621d8dc5a968217e4b44eaf59a8 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 3 Apr 2020 14:55:26 -0600 Subject: [PATCH 080/350] move out of tests/ --- Makefile | 2 +- tests/errors/main.cc => src/error-demo/error-demo.cc | 6 +++--- src/error-demo/local.mk | 10 ++++++++++ tests/errors/local.mk | 10 ---------- 4 files changed, 14 insertions(+), 14 deletions(-) rename tests/errors/main.cc => src/error-demo/error-demo.cc (95%) create mode 100644 src/error-demo/local.mk delete mode 100644 tests/errors/local.mk diff --git a/Makefile b/Makefile index 2d70aeb27..4fac27c47 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ makefiles = \ doc/manual/local.mk \ tests/local.mk \ tests/plugins/local.mk \ - tests/errors/local.mk + src/error-demo/local.mk -include Makefile.config diff --git a/tests/errors/main.cc b/src/error-demo/error-demo.cc similarity index 95% rename from tests/errors/main.cc rename to src/error-demo/error-demo.cc index cd6ca7caf..8fb20d38c 100644 --- a/tests/errors/main.cc +++ b/src/error-demo/error-demo.cc @@ -1,5 +1,5 @@ -#include "../../src/libutil/error.hh" -#include "../../src/libexpr/nixexpr.hh" +#include "error.hh" +#include "nixexpr.hh" #include #include @@ -9,7 +9,7 @@ int main() using namespace nix; // In each program where errors occur, this has to be set. - ErrorInfo::programName = std::optional("error-test"); + ErrorInfo::programName = std::optional("error-demo"); // There are currently four error types: // diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk new file mode 100644 index 000000000..121cab9e3 --- /dev/null +++ b/src/error-demo/local.mk @@ -0,0 +1,10 @@ +programs += error-demo + +error-demo_DIR := $(d) + +error-demo_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +error-demo_LIBS = libutil + +error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/tests/errors/local.mk b/tests/errors/local.mk deleted file mode 100644 index 0d64c8dbd..000000000 --- a/tests/errors/local.mk +++ /dev/null @@ -1,10 +0,0 @@ -programs += error-test - -error-test_DIR := $(d) - -error-test_SOURCES := \ - $(wildcard $(d)/*.cc) \ - -error-test_LIBS = libutil - -error-test_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system From 1221ae3dd07959d47d9f27e9d2271671003d2bed Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 5 Apr 2020 07:12:16 -0600 Subject: [PATCH 081/350] libexpr --- src/error-demo/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk index 121cab9e3..2cb2a0dc4 100644 --- a/src/error-demo/local.mk +++ b/src/error-demo/local.mk @@ -5,6 +5,6 @@ error-demo_DIR := $(d) error-demo_SOURCES := \ $(wildcard $(d)/*.cc) \ -error-demo_LIBS = libutil +error-demo_LIBS = libutil libexpr error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system From c976cb0b8ab5d0f2c4ab8c9826fc7db56e2f1b3e Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Sat, 4 Apr 2020 12:27:39 -0700 Subject: [PATCH 082/350] Don't retry on "unsupported protocol" error When encountering an unsupported protocol, there's no need to retry. Chances are, it won't suddenly be supported between retry attempts; error instead. Otherwise, you see something like the following: $ nix-env -i -f git://git@github.com/foo/bar warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 335 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 604 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 1340 ms warning: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1); retrying in 2685 ms With this change, you now see: $ nix-env -i -f git://git@github.com/foo/bar error: unable to download 'git://git@github.com/foo/bar': Unsupported protocol (1) --- src/libstore/download.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 149c84765..5967d0425 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -390,6 +390,7 @@ struct CurlDownloader : public Downloader case CURLE_SSL_CACERT_BADFILE: case CURLE_TOO_MANY_REDIRECTS: case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: err = Misc; break; default: // Shut up warnings From 85f14c4582ae3e64d7423ac48daaa0ea437fc144 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 6 Apr 2020 11:15:01 -0600 Subject: [PATCH 083/350] add libutil, libexpr include dirs --- src/error-demo/local.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk index 2cb2a0dc4..2c528490a 100644 --- a/src/error-demo/local.mk +++ b/src/error-demo/local.mk @@ -5,6 +5,8 @@ error-demo_DIR := $(d) error-demo_SOURCES := \ $(wildcard $(d)/*.cc) \ +error-demo_CXXFLAGS += -I src/libutil -I src/libexpr + error-demo_LIBS = libutil libexpr error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system From 2248cc6716ad5a9a56f9368e3cf6784a01cbb646 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 6 Apr 2020 12:05:17 -0600 Subject: [PATCH 084/350] ignore error-demo --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e3186fa76..ad5684123 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,8 @@ perl/Makefile.config /src/nix-copy-closure/nix-copy-closure +/src/error-demo/error-demo + /src/build-remote/build-remote # /tests/ From ec449c845056dad43eeec4d6ff983002f038cf69 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 6 Apr 2020 19:43:22 -0600 Subject: [PATCH 085/350] constructor style basically working --- src/error-demo/error-demo.cc | 71 +++++---- src/libutil/error.cc | 42 ++++- src/libutil/error.hh | 289 +++++++++++++++++------------------ 3 files changed, 222 insertions(+), 180 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 8fb20d38c..fdb574b25 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -22,21 +22,25 @@ int main() // created. // ProgramError takes name, description, and an optional hint. - printErrorInfo( ProgramError() - .name("name") - .description("error description") - .nohint() - ); + printErrorInfo( + ErrorInfo::ProgramError("name", + "error description", + std::nullopt)); // ProgramWarning takes name, description, and an optional hint. // The hint is in the form of a hintfmt class, which wraps boost::format(), // and makes all the substituted text yellow. - printErrorInfo( ProgramWarning() - .name("warning name") - .description("warning description") - // the templated value, 'warning', is automatically colored yellow. - .hint(hintfmt("there was a %1%", "warning")) - ); + printErrorInfo( + ErrorInfo::ProgramWarning("name", + "warning description", + std::optional(hintfmt("there was a %1%", "warning")))); + + // printErrorInfo( ProgramWarning() + // .name("warning name") + // .description("warning description") + // // the templated value, 'warning', is automatically colored yellow. + // .hint(hintfmt("there was a %1%", "warning")) + // ); /* // some invalid errors: @@ -63,25 +67,34 @@ int main() SymbolTable testTable; auto problem_symbol = testTable.create("problem"); - printErrorInfo(NixLangWarning() - .name("warning name") - .description("warning description") - .pos(Pos(problem_symbol, 40, 13)) - .linesOfCode(std::nullopt, - "this is the problem line of code", - std::nullopt) - .hint(hintfmt("this hint has %1% templated %2%!!", "yellow" , "values"))); + printErrorInfo(ErrorInfo::NixLangWarning( + "warning name", + "warning description", + Pos(problem_symbol, 40, 13), + std::nullopt, + "this is the problem line of code", + std::nullopt, + hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + + // // NixLangError is just the same as NixLangWarning, except for the Error + // // flag. + // printErrorInfo(NixLangError() + // .name("error name") + // .description("error description") + // .pos(Pos(problem_symbol, 40, 13)) + // .linesOfCode(std::optional("previous line of code"), + // "this is the problem line of code", + // std::optional("next line of code")) + // .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + printErrorInfo(ErrorInfo::NixLangError( + "error name", + "error description", + Pos(problem_symbol, 40, 13), + std::optional("previous line of code"), + "this is the problem line of code", + std::optional("next line of code"), + hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); - // NixLangError is just the same as NixLangWarning, except for the Error - // flag. - printErrorInfo(NixLangError() - .name("error name") - .description("error description") - .pos(Pos(problem_symbol, 40, 13)) - .linesOfCode(std::optional("previous line of code"), - "this is the problem line of code", - std::optional("next line of code")) - .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); return 0; } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8ab4a2dea..41fabbbcf 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -8,7 +8,43 @@ namespace nix std::optional ErrorInfo::programName = std::nullopt; -string showErrLine(ErrLine &errLine) +ErrorInfo ErrorInfo::ProgramError(const string &name, + const string &description, + const std::optional &hf) +{ + return ProgramEI(elError, name, description, hf); +} + +ErrorInfo ErrorInfo::ProgramWarning(const string &name, + const string &description, + const std::optional &hf) +{ + return ProgramEI(elWarning, name, description, hf); +} + + + +ErrorInfo ErrorInfo::ProgramEI(ErrLevel level, + const string &name, + const string &description, + const std::optional &hf) +{ + ErrorInfo ei(elError); + ei.name = name; + ei.description = description; + if (hf.has_value()) + ei.hint = std::optional(hf->str()); + else + ei.hint = std::nullopt; + return ei; +} + + + + + + +string showErrLine(const ErrLine &errLine) { if (errLine.columnRange.has_value()) { return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); @@ -17,7 +53,7 @@ string showErrLine(ErrLine &errLine) }; } -void printCodeLines(string &prefix, NixCode &nixCode) +void printCodeLines(const string &prefix, const NixCode &nixCode) { if (nixCode.errLine.has_value()) { @@ -69,7 +105,7 @@ void printCodeLines(string &prefix, NixCode &nixCode) } -void printErrorInfo(ErrorInfo &einfo) +void printErrorInfo(const ErrorInfo &einfo) { int errwidth = 80; string prefix = " "; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 57a9944be..a73b6639e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -41,153 +41,8 @@ class NixCode public: std::optional nixFile; std::optional errLine; - - ErrLine& ensureErrLine() - { - if (!this->errLine.has_value()) - this->errLine = std::optional(ErrLine()); - return *this->errLine; - } }; -// ------------------------------------------------- -// ErrorInfo. - -// Forward friend class declarations. "builder classes" -template -class AddName; - -template -class AddDescription; - -template -class AddPos; - -template -class AddLOC; - -// The error info class itself. -class ErrorInfo -{ -public: - ErrLevel level; - string name; - string description; - std::optional nixCode; - std::optional hint; - ErrorInfo& GetEI() - { - return *this; - } - - static std::optional programName; - - // give these access to the private constructor, - // when they are direct descendants (children but not grandchildren). - friend AddName; - friend AddDescription; - friend AddPos; - friend AddLOC; - - NixCode& ensureNixCode() - { - if (!this->nixCode.has_value()) - this->nixCode = std::optional(NixCode()); - return *this->nixCode; - } -protected: - // constructor is protected, so only the builder classes can create an ErrorInfo. - ErrorInfo(ErrLevel level) - { - this->level = level; - } -}; - -// Init as error -class EIError : public ErrorInfo -{ -protected: - EIError() : ErrorInfo(elError) {} -}; - -// Init as warning -class EIWarning : public ErrorInfo -{ -protected: - EIWarning() : ErrorInfo(elWarning) {} -}; - -// Builder class definitions. -template -class AddName : private T -{ -public: - T& name(const std::string &name) - { - GetEI().name = name; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddDescription : private T -{ -public: - T& description(const std::string &description) - { - GetEI().description = description; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddPos : private T -{ -public: - template - T& pos(const P &aPos) - { - GetEI().ensureNixCode().nixFile = aPos.file; - GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line; - GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 }; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddLOC : private T -{ -public: - T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) - { - GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; - GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; - GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - - // ---------------------------------------------------------------- // format for hints. same as fmt, except templated values // are always in yellow. @@ -229,7 +84,6 @@ public: friend class AddHint; private: format fmt; - }; template @@ -240,6 +94,143 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) return f; } +// ------------------------------------------------- +// ErrorInfo. +class ErrorInfo +{ +public: + ErrLevel level; + string name; + string description; + std::optional nixCode; + std::optional hint; + + static std::optional programName; + + static ErrorInfo ProgramError(const string &name, + const string &description, + const std::optional &hf); + + + static ErrorInfo ProgramWarning(const string &name, + const string &description, + const std::optional &hf); + + + template + static ErrorInfo NixLangError(const string &name, + const string &description, + const P &pos, + std::optional prevloc, + string loc, + std::optional nextloc, + const std::optional &hf) + { + return NixLangEI(elError, name, description, pos, prevloc, loc, nextloc, hf); + } + + + template + static ErrorInfo NixLangWarning(const string &name, + const string &description, + const P &pos, + std::optional prevloc, + string loc, + std::optional nextloc, + const std::optional &hf) + { + return NixLangEI(elWarning, name, description, pos, prevloc, loc, nextloc, hf); + } + + + +private: + template + static ErrorInfo NixLangEI(ErrLevel level, + const string &name, + const string &description, + const P &pos, + std::optional prevloc, + string loc, + std::optional nextloc, + const std::optional &hf) + { + ErrorInfo ei(level); + ei.name = name; + ei.description = description; + if (hf.has_value()) + ei.hint = std::optional(hf->str()); + else + ei.hint = std::nullopt; + + ErrLine errline; + errline.lineNumber = pos.line; + errline.columnRange = { .start = pos.column, .len = 1 }; + errline.prevLineOfCode = prevloc; + errline.errLineOfCode = loc; + errline.nextLineOfCode = nextloc; + NixCode nixcode; + nixcode.nixFile = pos.file; + nixcode.errLine = std::optional(errline); + ei.nixCode = std::optional(nixcode); + + return ei; + } + + static ErrorInfo ProgramEI(ErrLevel level, + const string &name, + const string &description, + const std::optional &hf); + + + + // constructor is protected, so only the builder classes can create an ErrorInfo. + ErrorInfo(ErrLevel level) + { + this->level = level; + } +}; + +/* +template +class AddPos : private T +{ +public: + template + T& pos(const P &aPos) + { + GetEI().ensureNixCode().nixFile = aPos.file; + GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line; + GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 }; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } +}; + +template +class AddLOC : private T +{ +public: + T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) + { + GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; + GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; + GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; + return *this; + } +protected: + ErrorInfo& GetEI() + { + return T::GetEI(); + } +}; +*/ + +/* // the template layer for adding a hint. template class AddHint : private T @@ -261,11 +252,12 @@ protected: return T::GetEI(); } }; +*/ // -------------------------------------------------------- // error types -typedef AddName< +/*typedef AddName< AddDescription< AddHint< EIError>>> ProgramError; @@ -290,11 +282,12 @@ typedef AddName< EIWarning>>>>> NixLangWarning; +*/ // -------------------------------------------------------- // error printing // just to cout for now. -void printErrorInfo(ErrorInfo &einfo); +void printErrorInfo(const ErrorInfo &einfo); } From 55c96b64e4de2b7e3443124bb0aa17ecc9188940 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 6 Apr 2020 20:14:48 -0600 Subject: [PATCH 086/350] comment cleanup --- src/error-demo/error-demo.cc | 84 +++++++++----------------------- src/libutil/error.hh | 92 ------------------------------------ 2 files changed, 23 insertions(+), 153 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index fdb574b25..ef8b56308 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -11,15 +11,10 @@ int main() // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); - // There are currently four error types: + // There are currently four constructor functions: // // ProgramError, ProgramWarning, NixLangError, NixLangWarning. // - // Each error type is created with a specific sequence of builder functions. - // Unlike with a constructor, each parameter is clearly named. - // If the sequence of function calls isn't followed, then there's a type - // error. This should make for a consistent look in the code when errors are - // created. // ProgramError takes name, description, and an optional hint. printErrorInfo( @@ -33,68 +28,35 @@ int main() printErrorInfo( ErrorInfo::ProgramWarning("name", "warning description", - std::optional(hintfmt("there was a %1%", "warning")))); - - // printErrorInfo( ProgramWarning() - // .name("warning name") - // .description("warning description") - // // the templated value, 'warning', is automatically colored yellow. - // .hint(hintfmt("there was a %1%", "warning")) - // ); - - /* - // some invalid errors: - - // type error: no hint function. - ProgramError() - .name("name") - .description("error description"); - - // type error: description before name. - ProgramError() - .description("error description") - .name("name") - .nohint(); - - // type error: hint function with regular boost format, not special - hintfmt. ProgramError() .description("error description") .name("name") - .hint(format("there was a %1%") % "warning"); - */ + std::optional( + hintfmt("there was a %1%", "warning")))); // NixLangWarning adds nix file, line number, column range, and the lines of // code where a warning occurred. - SymbolTable testTable; auto problem_symbol = testTable.create("problem"); - printErrorInfo(ErrorInfo::NixLangWarning( - "warning name", - "warning description", - Pos(problem_symbol, 40, 13), - std::nullopt, - "this is the problem line of code", - std::nullopt, - hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); - - // // NixLangError is just the same as NixLangWarning, except for the Error - // // flag. - // printErrorInfo(NixLangError() - // .name("error name") - // .description("error description") - // .pos(Pos(problem_symbol, 40, 13)) - // .linesOfCode(std::optional("previous line of code"), - // "this is the problem line of code", - // std::optional("next line of code")) - // .hint(hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); - printErrorInfo(ErrorInfo::NixLangError( - "error name", - "error description", - Pos(problem_symbol, 40, 13), - std::optional("previous line of code"), - "this is the problem line of code", - std::optional("next line of code"), - hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + printErrorInfo( + ErrorInfo::NixLangWarning( + "warning name", + "warning description", + Pos(problem_symbol, 40, 13), + std::nullopt, + "this is the problem line of code", + std::nullopt, + hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + // NixLangError is just the same as NixLangWarning, except for the Error + // flag. + printErrorInfo( + ErrorInfo::NixLangError( + "error name", + "error description", + Pos(problem_symbol, 40, 13), + std::optional("previous line of code"), + "this is the problem line of code", + std::optional("next line of code"), + hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); return 0; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a73b6639e..a8a1afbca 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -191,98 +191,6 @@ private: } }; -/* -template -class AddPos : private T -{ -public: - template - T& pos(const P &aPos) - { - GetEI().ensureNixCode().nixFile = aPos.file; - GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line; - GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 }; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; - -template -class AddLOC : private T -{ -public: - T& linesOfCode(std::optional prevloc, string loc, std::optional nextloc) - { - GetEI().ensureNixCode().ensureErrLine().prevLineOfCode = prevloc; - GetEI().ensureNixCode().ensureErrLine().errLineOfCode = loc; - GetEI().ensureNixCode().ensureErrLine().nextLineOfCode = nextloc; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; -*/ - -/* -// the template layer for adding a hint. -template -class AddHint : private T -{ -public: - T& hint(const hintformat &hf) - { - GetEI().hint = std::optional(hf.str()); - return *this; - } - T& nohint() - { - GetEI().hint = std::nullopt; - return *this; - } -protected: - ErrorInfo& GetEI() - { - return T::GetEI(); - } -}; -*/ - -// -------------------------------------------------------- -// error types - -/*typedef AddName< - AddDescription< - AddHint< - EIError>>> ProgramError; - -typedef AddName< - AddDescription< - AddHint< - EIWarning>>> ProgramWarning; - -typedef AddName< - AddDescription< - AddPos< - AddLOC< - AddHint< - EIError>>>>> NixLangError; - -typedef AddName< - AddDescription< - AddPos< - AddLOC< - AddHint< - EIWarning>>>>> NixLangWarning; - - -*/ // -------------------------------------------------------- // error printing From 462421d34588c227eb736b07f7b40a38aa0972a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 16:04:18 +0200 Subject: [PATCH 087/350] Backport libfetchers from the flakes branch This provides a pluggable mechanism for defining new fetchers. It adds a builtin function 'fetchTree' that generalizes existing fetchers like 'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a set of attributes, e.g. fetchTree { type = "git"; url = "https://example.org/repo.git"; ref = "some-branch"; rev = "abcdef..."; } The existing fetchers are just wrappers around this. Note that the input attributes to fetchTree are the same as flake input specifications and flake lock file entries. All fetchers share a common cache stored in ~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}). This also adds support for Git worktrees (c169ea59049f861aaba429f48b828d0820b74d1d). --- Makefile | 1 + src/libexpr/common-eval-args.cc | 8 +- src/libexpr/local.mk | 4 +- src/libexpr/parser.y | 6 +- src/libexpr/primops.cc | 67 ----- src/libexpr/primops.hh | 1 + src/libexpr/primops/fetchGit.cc | 223 ++------------ src/libexpr/primops/fetchMercurial.cc | 207 +++---------- src/libexpr/primops/fetchTree.cc | 165 +++++++++++ src/libfetchers/attrs.cc | 92 ++++++ src/libfetchers/attrs.hh | 37 +++ src/libfetchers/cache.cc | 121 ++++++++ src/libfetchers/cache.hh | 34 +++ src/libfetchers/fetchers.cc | 73 +++++ src/libfetchers/fetchers.hh | 103 +++++++ src/libfetchers/git.cc | 401 ++++++++++++++++++++++++++ src/libfetchers/github.cc | 195 +++++++++++++ src/libfetchers/local.mk | 11 + src/libfetchers/mercurial.cc | 303 +++++++++++++++++++ src/libfetchers/tarball.cc | 277 ++++++++++++++++++ src/libfetchers/tree-info.cc | 14 + src/libfetchers/tree-info.hh | 29 ++ src/libstore/derivations.cc | 2 +- src/libstore/download.cc | 137 --------- src/libstore/download.hh | 31 +- src/libstore/globals.hh | 9 + src/libstore/store-api.cc | 25 +- src/libstore/store-api.hh | 1 + src/libutil/types.hh | 8 + src/libutil/url.cc | 137 +++++++++ src/libutil/url.hh | 62 ++++ src/nix-channel/nix-channel.cc | 17 +- src/nix/local.mk | 4 +- tests/fetchGit.sh | 32 +- tests/init.sh | 2 +- tests/tarball.sh | 7 + 36 files changed, 2199 insertions(+), 647 deletions(-) create mode 100644 src/libexpr/primops/fetchTree.cc create mode 100644 src/libfetchers/attrs.cc create mode 100644 src/libfetchers/attrs.hh create mode 100644 src/libfetchers/cache.cc create mode 100644 src/libfetchers/cache.hh create mode 100644 src/libfetchers/fetchers.cc create mode 100644 src/libfetchers/fetchers.hh create mode 100644 src/libfetchers/git.cc create mode 100644 src/libfetchers/github.cc create mode 100644 src/libfetchers/local.mk create mode 100644 src/libfetchers/mercurial.cc create mode 100644 src/libfetchers/tarball.cc create mode 100644 src/libfetchers/tree-info.cc create mode 100644 src/libfetchers/tree-info.hh create mode 100644 src/libutil/url.cc create mode 100644 src/libutil/url.hh diff --git a/Makefile b/Makefile index 469070533..e3057c36c 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ makefiles = \ nix-rust/local.mk \ src/libutil/local.mk \ src/libstore/local.mk \ + src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 13950ab8d..82bfeac36 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -3,6 +3,8 @@ #include "download.hh" #include "util.hh" #include "eval.hh" +#include "fetchers.hh" +#include "store-api.hh" namespace nix { @@ -46,9 +48,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a4ccab376..917e8a1c7 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,9 +6,9 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -libexpr_LIBS = libutil libstore libnixrust +libexpr_LIBS = libutil libstore libfetchers libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..a30fb44b5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -545,6 +545,7 @@ formal #include "eval.hh" #include "download.hh" +#include "fetchers.hh" #include "store-api.hh" @@ -687,9 +688,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - CachedDownloadRequest request(elem.second); - request.unpack = true; - res = { true, getDownloader()->downloadCached(store, request).path }; + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..629f3da15 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -2045,68 +2044,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } -/************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } - - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); - - } else - request.uri = state.forceStringNoCtx(*args[0], pos); - - state.checkURI(request.uri); - - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - - auto res = getDownloader()->downloadCached(state.store, request); - - if (state.allowedPaths) - state.allowedPaths->insert(res.path); - - mkString(v, res.storePath, PathSet({res.storePath})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); -} - - /************************************************************* * Primop registration *************************************************************/ @@ -2289,10 +2226,6 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index c790b30f6..05d0792ef 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -20,6 +20,7 @@ struct RegisterPrimOp them. */ /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); + /* Execute a program and parse its output */ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..812de9d91 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,202 +1,17 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" #include "hash.hh" -#include "tarfile.hh" - -#include - -#include - -#include - -using namespace std::string_literals; +#include "fetchers.hh" +#include "url.hh" namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - -std::regex revRegex("^[0-9a-fA-F]{40}$"); - -GitInfo exportGit(ref store, const std::string & uri, - std::optional ref, std::string rev, - const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { - - bool clean = true; - - try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return gitInfo; - } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; - } - - if (!ref) ref = "HEAD"s; - - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - - deletePath(getCacheDir() + "/nix/git"); - - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); - - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } - - Path localRefFile; - if (ref->compare(0, 5, "refs/") == 0) - localRefFile = cacheDir + "/" + *ref; - else - localRefFile = cacheDir + "/refs/heads/" + *ref; - - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); - - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); - } - - // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == gitInfo.rev); - - gitInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) { - gitInfo.revCount = json["revCount"]; - return gitInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - unpackTarfile(*source, tmpDir); - - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); - - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; - - writeFile(storeLink, json.dump()); - - return gitInfo; -} - static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; std::optional ref; - std::string rev; + std::optional rev; std::string name = "source"; PathSet context; @@ -213,7 +28,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -230,17 +45,35 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "git+" + url + : "git+file://" + url); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); + // Backward compatibility: set 'revCount' to 0 for a dirty tree. + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + tree.info.revCount.value_or(0)); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..f18351646 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,174 +1,18 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" - -#include +#include "fetchers.hh" +#include "url.hh" #include -#include - -using namespace std::string_literals; - namespace nix { -struct HgInfo -{ - Path storePath; - std::string branch; - std::string rev; - uint64_t revCount = 0; -}; - -std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); - -HgInfo exportMercurial(ref store, const std::string & uri, - std::string rev, const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - printTalkative("copying unclean Mercurial working tree '%s'", uri); - - HgInfo hgInfo; - hgInfo.rev = "0000000000000000000000000000000000000000"; - hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); - - auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return hgInfo; - } - } - - if (rev == "") rev = "default"; - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); - - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); - - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) - { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(std::regex_match(rev, commitHashRegex) - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); - } - } - - writeFile(stampFile, ""); - } - - auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); - - HgInfo hgInfo; - hgInfo.rev = tokens[0]; - hgInfo.revCount = std::stoull(tokens[1]); - hgInfo.branch = tokens[2]; - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == hgInfo.rev); - - hgInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) { - printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); - return hgInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); - - deletePath(tmpDir + "/.hg_archival.txt"); - - hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - nlohmann::json json; - json["storePath"] = hgInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["branch"] = hgInfo.branch; - json["rev"] = hgInfo.rev; - json["revCount"] = hgInfo.revCount; - - writeFile(storeLink, json.dump()); - - return hgInfo; -} - static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::string rev; + std::optional rev; + std::optional ref; std::string name = "source"; PathSet context; @@ -182,8 +26,15 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar string n(attr.name); if (n == "url") url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") { + // Ugly: unlike fetchGit, here the "rev" attribute can + // be both a revision or a branch/tag name. + auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + if (std::regex_match(value, revRegex)) + rev = Hash(value, htSHA1); + else + ref = value; + } else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -200,18 +51,36 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); - auto hgInfo = exportMercurial(state.store, url, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "hg+" + url + : "hg+file://" + url); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); - mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + if (input2->getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 000000000..9586f71ed --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,165 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "download.hh" + +#include +#include + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to '%s', at %s", + attr.name, who, attr.pos); + } + + if (!url) + throw EvalError("'url' argument required, at %s", pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(), hash.to_string()); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc new file mode 100644 index 000000000..40c02de42 --- /dev/null +++ b/src/libfetchers/attrs.cc @@ -0,0 +1,92 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if>(&attr.second)) { + json[attr.first] = v->t; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not an integer", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +} diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh new file mode 100644 index 000000000..2c9e772d2 --- /dev/null +++ b/src/libfetchers/attrs.hh @@ -0,0 +1,37 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; +}; + +typedef std::variant> Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc new file mode 100644 index 000000000..e1c7f3dee --- /dev/null +++ b/src/libfetchers/cache.cc @@ -0,0 +1,121 @@ +#include "cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh new file mode 100644 index 000000000..d76ab1233 --- /dev/null +++ b/src/libfetchers/cache.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional lookupExpired( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc new file mode 100644 index 000000000..25d3da431 --- /dev/null +++ b/src/libfetchers/fetchers.cc @@ -0,0 +1,73 @@ +#include "fetchers.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::unique_ptr inputFromAttrs(const Attrs & attrs) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs); + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = Hash(*narHash); + return res; + } + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("type", type()); + return attrs; +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(tree.storePath); + + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.info.narHash); + + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + + return {std::move(tree), input}; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh new file mode 100644 index 000000000..59a58ae67 --- /dev/null +++ b/src/libfetchers/fetchers.hh @@ -0,0 +1,103 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" +#include "tree-info.hh" +#include "attrs.hh" +#include "url.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + TreeInfo info; +}; + +struct Input : std::enable_shared_from_this +{ + std::optional narHash; // FIXME: implement + + virtual std::string type() const = 0; + + virtual ~Input() { } + + virtual bool operator ==(const Input & other) const { return false; } + + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + virtual bool isDirect() const { return true; } + + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } + + Attrs toAttrs() const; + + std::pair> fetchTree(ref store) const; + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; +}; + +struct InputScheme +{ + virtual ~InputScheme() { } + + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +std::unique_ptr inputFromAttrs(const Attrs & attrs); + +void registerInputScheme(std::unique_ptr && fetcher); + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 000000000..cee0713f4 --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,401 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + bool shallow = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + return attrs; + } + + std::pair getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc new file mode 100644 index 000000000..3fc95ff51 --- /dev/null +++ b/src/libfetchers/github.cc @@ -0,0 +1,195 @@ +#include "download.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + std::string type() const override { return "github"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 000000000..d7143d8a6 --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,11 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_CXXFLAGS += -I src/libutil -I src/libstore + +libfetchers_LIBS = libutil libstore libnixrust diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc new file mode 100644 index 000000000..790f7c753 --- /dev/null +++ b/src/libfetchers/mercurial.cc @@ -0,0 +1,303 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + MercurialInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "hg"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair getActualUrl() const + { + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // FIXME: return lastModified. + + // FIXME: don't clone local repositories. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", (int64_t) revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc new file mode 100644 index 000000000..b0a83001e --- /dev/null +++ b/src/libfetchers/tarball.cc @@ -0,0 +1,277 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "download.hh" +#include "globals.hh" +#include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" + +namespace nix::fetchers { + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + auto useCached = [&]() -> DownloadFileResult + { + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + }; + }; + + if (cached && !cached->expired) + return useCached(); + + DownloadRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + DownloadResult res; + try { + res = getDownloader()->download(request); + } catch (DownloadError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } + + // FIXME: write to temporary file. + + Attrs infoAttrs({ + {"etag", res.etag}, + {"url", res.effectiveUri}, + }); + + std::optional storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + + getCache()->add( + store, + inAttrs, + infoAttrs, + *storePath, + immutable); + + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + + return { + .storePath = std::move(*storePath), + .etag = res.etag, + .effectiveUrl = res.effectiveUri, + }; +} + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) + return Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), + .info = TreeInfo { + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + }, + }; + + auto res = downloadFile(store, url, name, immutable); + + std::optional unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + } + + Attrs infoAttrs({ + {"lastModified", lastModified}, + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + *unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + +struct TarballInput : Input +{ + ParsedURL url; + std::optional hash; + + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return hash || narHash; + } + + ParsedURL toURL() const override + { + auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI)); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + else if (hash) + attrs.emplace("hash", hash->to_string(SRI)); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto tree = downloadTarball(store, url.to_string(), "source", false); + + auto input = std::make_shared(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; + + return {std::move(tree), input}; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique(url); + + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { + // FIXME: require SRI hash. + input->hash = Hash(hash->second); + input->url.query.erase(hash); + } + + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash" && name != "narHash") + throw Error("unsupported tarball input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + // FIXME: require SRI hash. + input->hash = Hash(*hash); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc new file mode 100644 index 000000000..5788e94a1 --- /dev/null +++ b/src/libfetchers/tree-info.cc @@ -0,0 +1,14 @@ +#include "tree-info.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(true, narHash, "source"); +} + +} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh new file mode 100644 index 000000000..2c7347281 --- /dev/null +++ b/src/libfetchers/tree-info.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct TreeInfo +{ + Hash narHash; + std::optional revCount; + std::optional lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && revCount == other.revCount + && lastModified == other.lastModified; + } + + StorePath computeStorePath(Store & store) const; +}; + +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ddc86a..0067032af 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; + readDerivation(store, store.toRealPath(i.first)), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5967d0425..215046b72 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -35,10 +35,6 @@ DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } - std::string resolveUri(const std::string & uri) { if (uri.compare(0, 8, "channel:") == 0) @@ -802,139 +798,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - // FIXME - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first - : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - bool isUri(const string & s) { diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 5a131c704..28c8a9162 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -65,28 +65,6 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional etag; - std::string effectiveUri; -}; - class Store; struct Downloader @@ -108,12 +86,6 @@ struct Downloader invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); - enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; @@ -135,4 +107,7 @@ public: bool isUri(const string & s); +/* Resolve deprecated 'channel:' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3aa3653f3..40f350f0b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -351,12 +351,21 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + Setting githubAccessToken{this, "", "github-access-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b9e894a9a..e5282bb30 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "url.hh" #include @@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } @@ -866,27 +867,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0fa59be6a..81014763d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -28,6 +28,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); +MakeError(NotInStore, Error); struct BasicDerivation; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 20b96a85c..a1ce7b372 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -157,4 +157,12 @@ typedef list Paths; typedef set PathSet; +/* Helper class to run code at startup. */ +template +struct OnStartup +{ + OnStartup(T && t) { t(); } +}; + + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc new file mode 100644 index 000000000..5d5328e5d --- /dev/null +++ b/src/libutil/url.cc @@ -0,0 +1,137 @@ +#include "url.hh" +#include "util.hh" + +namespace nix { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[1]; + std::string scheme = match[2]; + auto authority = match[3].matched + ? std::optional(match[3]) : std::nullopt; + std::string path = match[4].matched ? match[4] : match[5]; + auto & query = match[6]; + auto & fragment = match[7]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map decodeQuery(const std::string & query) +{ + std::map result; + + for (auto s : tokenizeString(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +bool ParsedURL::operator ==(const ParsedURL & other) const +{ + return + scheme == other.scheme + && authority == other.authority + && path == other.path + && query == other.query + && fragment == other.fragment; +} + +} diff --git a/src/libutil/url.hh b/src/libutil/url.hh new file mode 100644 index 000000000..1503023a2 --- /dev/null +++ b/src/libutil/url.hh @@ -0,0 +1,62 @@ +#pragma once + +#include "types.hh" + +#include + +namespace nix { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index a2639579d..2a9defb4e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -3,6 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "../nix/legacy.hh" +#include "fetchers.hh" #include #include @@ -86,12 +87,9 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - CachedDownloadRequest request(url); - request.ttl = 0; - auto dl = getDownloader(); - auto result = dl->downloadCached(store, request); - auto filename = result.path; - url = chomp(result.effectiveUri); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -114,11 +112,10 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); } catch (DownloadError & e) { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } - chomp(filename); } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -185,6 +182,8 @@ static int _main(int argc, char ** argv) } else if (*arg == "--rollback") { cmd = cRollback; } else { + if (hasPrefix(*arg, "-")) + throw UsageError("unsupported argument '%s'", *arg); args.push_back(std::move(*arg)); } return true; diff --git a/src/nix/local.mk b/src/nix/local.mk index 50a18efd7..033675e89 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,9 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index ed8fa14d6..d9c9874f5 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -11,7 +11,7 @@ repo=$TEST_ROOT/git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow git init $repo git -C $repo config user.email "foobar@example.com" @@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD) echo world > $repo/hello git -C $repo commit -m 'Bla2' -a +git -C $repo worktree add $TEST_ROOT/worktree +echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) +# Fetch a worktree +unset _NIX_FORCE_HTTP +path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +export _NIX_FORCE_HTTP=1 +[[ $(tail -n 1 $path0/hello) = "hello" ]] + # Fetch the default branch. path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] @@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -# But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) - # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] @@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar git -C $repo add dir1/foo git -C $repo rm hello +unset _NIX_FORCE_HTTP path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] @@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") git -C $repo checkout $rev2 -b dev echo dev > $repo/hello -# File URI uses 'master' unless specified otherwise +# File URI uses dirty tree unless specified otherwise path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") -[[ $path = $path2 ]] +[ $(cat $path2/hello) = dev ] # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") @@ -131,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP # Nuke the cache -rm -rf $TEST_HOME/.cache/nix/gitv2 +rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH +# Try again, but without 'git' on PATH. This should fail. NIX=$(command -v nix) # This should fail (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) @@ -141,3 +147,13 @@ NIX=$(command -v nix) # Try again, with 'git' available. This should work. path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] + +# Fetching a shallow repo shouldn't work by default, because we can't +# return a revCount. +git clone --depth 1 file://$repo $TEST_ROOT/shallow +(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") + +# But you can request a shallow clone, which won't return a revCount. +path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") +[[ $path3 = $path6 ]] +[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] diff --git a/tests/init.sh b/tests/init.sh index 6a119aad0..c62c4856a 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <&1 | grep 'NAR hash mismatch in input' + nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) From 670feb000a9fac76f0996711f061ec466a53dc97 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 14:56:20 +0200 Subject: [PATCH 088/350] Add 'path' fetcher This fetchers copies a plain directory (i.e. not a Git/Mercurial repository) to the store (or does nothing if the path is already a store path). One use case is to pin the 'nixpkgs' flake used to build the current NixOS system, and prevent it from being garbage-collected, via a system registry entry like this: { "from": { "id": "nixpkgs", "type": "indirect" }, "to": { "type": "path", "path": "/nix/store/rralhl3wj4rdwzjn16g7d93mibvlr521-source", "lastModified": 1585388205, "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a" }, "exact": true } Note the fake "lastModified" and "rev" attributes that ensure that the flake gives the same evaluation results as the corresponding Git/GitHub inputs. (cherry picked from commit 12f9379123eba828f2ae06f7978a37b7045c2b23) --- src/libfetchers/attrs.cc | 15 +++++ src/libfetchers/attrs.hh | 2 + src/libfetchers/path.cc | 130 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 src/libfetchers/path.cc diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 40c02de42..feb0a6085 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -89,4 +89,19 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name) return *s; } +std::map attrsToQuery(const Attrs & attrs) +{ + std::map query; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, fmt("%d", *v)); + } else if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, *v); + } else if (auto v = std::get_if>(&attr.second)) { + query.insert_or_assign(attr.first, v->t ? "1" : "0"); + } else abort(); + } + return query; +} + } diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 2c9e772d2..d6e0ae000 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -34,4 +34,6 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na bool getBoolAttr(const Attrs & attrs, const std::string & name); +std::map attrsToQuery(const Attrs & attrs); + } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc new file mode 100644 index 000000000..9801d9b86 --- /dev/null +++ b/src/libfetchers/path.cc @@ -0,0 +1,130 @@ +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct PathInput : Input +{ + Path path; + + /* Allow the user to pass in "fake" tree info attributes. This is + useful for making a pinned tree work the same as the repository + from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + std::optional rev; + std::optional revCount; + std::optional lastModified; + + std::string type() const override { return "path"; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto query = attrsToQuery(toAttrsInternal()); + query.erase("path"); + return ParsedURL { + .scheme = "path", + .path = path, + .query = query, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("path", path); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (revCount) + attrs.emplace("revCount", *revCount); + if (lastModified) + attrs.emplace("lastModified", *lastModified); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto input = std::make_shared(*this); + + auto storePath = store->maybeParseStorePath(path); + + if (storePath) + store->addTempRoot(*storePath); + + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + // FIXME: try to substitute storePath. + storePath = store->addToStore("name", path); + + return + { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = revCount, + .lastModified = lastModified + } + }, + input + }; + } + +}; + +struct PathInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return nullptr; + + auto input = std::make_unique(); + input->path = url.path; + + for (auto & [name, value] : url.query) + if (name == "rev") + input->rev = Hash(value, htSHA1); + else if (name == "revCount") { + uint64_t revCount; + if (!string2Int(value, revCount)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->revCount = revCount; + } + else if (name == "lastModified") { + time_t lastModified; + if (!string2Int(value, lastModified)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->lastModified = lastModified; + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + auto input = std::make_unique(); + input->path = getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + if (name == "rev") + input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + else if (name == "revCount") + input->revCount = getIntAttr(attrs, "revCount"); + else if (name == "lastModified") + input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "type" || name == "path") + ; + else + throw Error("unsupported path input attribute '%s'", name); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} From f58a9b0e62879e28cc7cac59b489a52c924bbc36 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:33 +0200 Subject: [PATCH 089/350] Respect the narHash attribute in more input types (cherry picked from commit a6ff66b658b61aef80d936f0183447fe4cb46000) --- src/libfetchers/fetchers.cc | 4 +++- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 6 ++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 25d3da431..94ac30e38 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -29,8 +29,10 @@ std::unique_ptr inputFromURL(const std::string & url) std::unique_ptr inputFromAttrs(const Attrs & attrs) { + auto attrs2(attrs); + attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(attrs2); if (res) { if (auto narHash = maybeGetStrAttr(attrs, "narHash")) // FIXME: require SRI hash. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index cee0713f4..3f94d9bdd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -39,7 +39,7 @@ struct GitInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 3fc95ff51..ef27eaa76 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -33,7 +33,7 @@ struct GitHubInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 790f7c753..1d6571571 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -33,7 +33,7 @@ struct MercurialInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b0a83001e..4c4e5828e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -205,9 +205,7 @@ struct TarballInput : Input { Attrs attrs; attrs.emplace("url", url.to_string()); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - else if (hash) + if (hash) attrs.emplace("hash", hash->to_string(SRI)); return attrs; } @@ -260,7 +258,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash" && name != "narHash") + if (name != "type" && name != "url" && name != "hash") throw Error("unsupported tarball input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); From a6dfa3cb85b59a0979de7fe02c9d67939c4ac217 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:27 +0200 Subject: [PATCH 090/350] PathInput: Add some methods (cherry picked from commit 78ad5b3d91507427fa563f3474dc52da608ad224) --- src/libfetchers/path.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9801d9b86..037404726 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -19,6 +19,22 @@ struct PathInput : Input std::optional getRev() const override { return rev; } + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && path == other2->path + && rev == other2->rev + && revCount == other2->revCount + && lastModified == other2->lastModified; + } + + bool isImmutable() const override + { + return (bool) narHash; + } + ParsedURL toURL() const override { auto query = attrsToQuery(toAttrsInternal()); From 26aeeb7653fa051ddec913bdc2c578b9066bc08f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Apr 2020 14:28:37 +0200 Subject: [PATCH 091/350] Add FIXME (cherry picked from commit 2f494531b7811b45f6b76787f225495a14d28a7f) --- src/libfetchers/path.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 037404726..7c7e20f4e 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -63,6 +63,8 @@ struct PathInput : Input { auto input = std::make_shared(*this); + // FIXME: check whether access to 'path' is allowed. + auto storePath = store->maybeParseStorePath(path); if (storePath) From cd39709003eee4b85a31799b8bc2de59721930ce Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Apr 2020 09:27:17 +0200 Subject: [PATCH 092/350] Cleanup --- src/libstore/download.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 215046b72..af69699a8 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -1,14 +1,10 @@ #include "download.hh" #include "util.hh" #include "globals.hh" -#include "hash.hh" #include "store-api.hh" -#include "archive.hh" #include "s3.hh" #include "compression.hh" -#include "pathlocks.hh" #include "finally.hh" -#include "tarfile.hh" #ifdef ENABLE_S3 #include From 9d04b5da17898fd564308423bc7d10fc929efe18 Mon Sep 17 00:00:00 2001 From: mlatus Date: Tue, 7 Apr 2020 20:29:40 +0800 Subject: [PATCH 093/350] `nix run` using $SHELL as default command --- src/nix/run.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/run.cc b/src/nix/run.cc index 8e30264c0..5334531fd 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -59,14 +59,14 @@ struct RunCommon : virtual Command struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment { - std::vector command = { "bash" }; + std::vector command = { getEnv("SHELL").value_or("bash") }; CmdRun() { mkFlag() .longName("command") .shortName('c') - .description("command and arguments to be executed; defaults to 'bash'") + .description("command and arguments to be executed; defaults to '$SHELL'") .labels({"command", "args"}) .arity(ArityAny) .handler([&](std::vector ss) { From 20c0984a46b6fbc1be0b073f14f5e20b7c82a26a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 7 Apr 2020 10:14:15 -0600 Subject: [PATCH 094/350] remove columnrange; switch to fmt in error.cc --- src/libutil/error.cc | 68 ++++++++++++++++++++++---------------------- src/libutil/error.hh | 11 ++----- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8ab4a2dea..106ea127e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,10 +10,10 @@ std::optional ErrorInfo::programName = std::nullopt; string showErrLine(ErrLine &errLine) { - if (errLine.columnRange.has_value()) { - return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); + if (errLine.column > 0) { + return fmt("(%1%:%2%)", errLine.lineNumber, errLine.column); } else { - return (format("(%1%)") % errLine.lineNumber).str(); + return fmt("(%1%)", errLine.lineNumber); }; } @@ -23,45 +23,45 @@ void printCodeLines(string &prefix, NixCode &nixCode) if (nixCode.errLine.has_value()) { // previous line of code. if (nixCode.errLine->prevLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber - 1) - % *nixCode.errLine->prevLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errLine->lineNumber - 1), + *nixCode.errLine->prevLineOfCode) << std::endl; } // line of code containing the error.%2$+5d% - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber) - % nixCode.errLine->errLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errLine->lineNumber), + nixCode.errLine->errLineOfCode) << std::endl; // error arrows for the column range. - if (nixCode.errLine->columnRange.has_value()) { - int start = nixCode.errLine->columnRange->start; + if (nixCode.errLine->column > 0) { + int start = nixCode.errLine->column; std::string spaces; for (int i = 0; i < start; ++i) { spaces.append(" "); } - int len = nixCode.errLine->columnRange->len; - std::string arrows; - for (int i = 0; i < len; ++i) { - arrows.append("^"); - } + // for now, length of 1. + std::string arrows("^"); - std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; + std::cout << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows) << std::endl; } // next line of code. if (nixCode.errLine->nextLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber + 1) - % *nixCode.errLine->nextLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errLine->lineNumber + 1), + *nixCode.errLine->nextLineOfCode) << std::endl; } @@ -89,7 +89,7 @@ void printErrorInfo(ErrorInfo &einfo) break; } default: { - levelString = (format("invalid error level: %1%") % einfo.level).str(); + levelString = fmt("invalid error level: %1%", einfo.level); break; } } @@ -102,13 +102,13 @@ void printErrorInfo(ErrorInfo &einfo) dashes.append("-"); // divider. - std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) - % prefix - % levelString - % "---" - % einfo.name - % dashes - % einfo.programName.value_or("") + std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL + , prefix + , levelString + , "---" + , einfo.name + , dashes + , einfo.programName.value_or("")) << std::endl; // filename. @@ -118,11 +118,11 @@ void printErrorInfo(ErrorInfo &einfo) ? string(" ") + showErrLine(*einfo.nixCode->errLine) : ""; - std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % *einfo.nixCode->nixFile % eline << std::endl; + std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL + , prefix, *einfo.nixCode->nixFile, eline) << std::endl; std::cout << prefix << std::endl; } else { - std::cout << format("%1%from command line argument") % prefix << std::endl; + std::cout << fmt("%1%from command line argument", prefix) << std::endl; std::cout << prefix << std::endl; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 57a9944be..664bfbb5a 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -17,20 +17,13 @@ typedef enum { elError } ErrLevel; -class ColumnRange -{ -public: - unsigned int start; - unsigned int len; -}; - class ErrorInfo; class ErrLine { public: int lineNumber; - std::optional columnRange; + int column; std::optional prevLineOfCode; string errLineOfCode; std::optional nextLineOfCode; @@ -159,7 +152,7 @@ public: { GetEI().ensureNixCode().nixFile = aPos.file; GetEI().ensureNixCode().ensureErrLine().lineNumber = aPos.line; - GetEI().ensureNixCode().ensureErrLine().columnRange = { .start = aPos.column, .len = 1 }; + GetEI().ensureNixCode().ensureErrLine().column = aPos.column; return *this; } protected: From 00c507cc52ceb0d879c43e88e70de0028ff47fc6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 7 Apr 2020 14:36:32 -0600 Subject: [PATCH 095/350] columnRange -> column --- src/libutil/error.cc | 14 +++++--------- src/libutil/error.hh | 13 ++++--------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 41fabbbcf..db8821a5c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -46,8 +46,8 @@ ErrorInfo ErrorInfo::ProgramEI(ErrLevel level, string showErrLine(const ErrLine &errLine) { - if (errLine.columnRange.has_value()) { - return (format("(%1%:%2%)") % errLine.lineNumber % errLine.columnRange->start).str(); + if (errLine.column > 0) { + return (format("(%1%:%2%)") % errLine.lineNumber % errLine.column).str(); } else { return (format("(%1%)") % errLine.lineNumber).str(); }; @@ -74,18 +74,14 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) << std::endl; // error arrows for the column range. - if (nixCode.errLine->columnRange.has_value()) { - int start = nixCode.errLine->columnRange->start; + if (nixCode.errLine->column > 0) { + int start = nixCode.errLine->column; std::string spaces; for (int i = 0; i < start; ++i) { spaces.append(" "); } - int len = nixCode.errLine->columnRange->len; - std::string arrows; - for (int i = 0; i < len; ++i) { - arrows.append("^"); - } + std::string arrows("^"); std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a8a1afbca..e3bb2c1dd 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -17,20 +17,13 @@ typedef enum { elError } ErrLevel; -class ColumnRange -{ -public: - unsigned int start; - unsigned int len; -}; - class ErrorInfo; class ErrLine { public: int lineNumber; - std::optional columnRange; + int column; std::optional prevLineOfCode; string errLineOfCode; std::optional nextLineOfCode; @@ -107,6 +100,8 @@ public: static std::optional programName; + ErrorInfo& set_name(const string &name) { this->name = name; return *this; } + static ErrorInfo ProgramError(const string &name, const string &description, const std::optional &hf); @@ -165,7 +160,7 @@ private: ErrLine errline; errline.lineNumber = pos.line; - errline.columnRange = { .start = pos.column, .len = 1 }; + errline.column = pos.column; errline.prevLineOfCode = prevloc; errline.errLineOfCode = loc; errline.nextLineOfCode = nextloc; From 1ab8d6ac1861e9405ae34af3deb681020c03e82d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Apr 2020 15:18:41 +0200 Subject: [PATCH 096/350] Downloader: Only write data to the sink on a 200 response Hopefully fixes #3278. --- src/libstore/download.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index af69699a8..bc875f653 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -83,8 +83,15 @@ struct CurlDownloader : public Downloader , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } } else this->result.data->append((char *) data, len); }) From 47ed067d45d3de7786cdb55f187b0db2eb6289c1 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 8 Apr 2020 09:07:58 -0600 Subject: [PATCH 097/350] initializer style --- src/error-demo/error-demo.cc | 22 +++--- src/libutil/error.cc | 85 ++++++--------------- src/libutil/error.hh | 144 +++++++++++++---------------------- 3 files changed, 91 insertions(+), 160 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index ef8b56308..7eef0b162 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -18,22 +18,26 @@ int main() // ProgramError takes name, description, and an optional hint. printErrorInfo( - ErrorInfo::ProgramError("name", - "error description", - std::nullopt)); + ErrorInfo { .level = elError, + .name = "name", + .description = "error description", + }); // ProgramWarning takes name, description, and an optional hint. // The hint is in the form of a hintfmt class, which wraps boost::format(), // and makes all the substituted text yellow. printErrorInfo( - ErrorInfo::ProgramWarning("name", - "warning description", - std::optional( - hintfmt("there was a %1%", "warning")))); + ErrorInfo { .level = elWarning, + .name = "name", + .description = "error description", + .hint = std::optional( + hintfmt("there was a %1%", "warning")) + }); + // NixLangWarning adds nix file, line number, column range, and the lines of // code where a warning occurred. - SymbolTable testTable; +/* SymbolTable testTable; auto problem_symbol = testTable.create("problem"); printErrorInfo( @@ -58,5 +62,5 @@ int main() std::optional("next line of code"), hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); - return 0; +*/ return 0; } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index db8821a5c..24ed4df2e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -8,74 +8,37 @@ namespace nix std::optional ErrorInfo::programName = std::nullopt; -ErrorInfo ErrorInfo::ProgramError(const string &name, - const string &description, - const std::optional &hf) +string showErrPos(const ErrPos &errPos) { - return ProgramEI(elError, name, description, hf); -} - -ErrorInfo ErrorInfo::ProgramWarning(const string &name, - const string &description, - const std::optional &hf) -{ - return ProgramEI(elWarning, name, description, hf); -} - - - -ErrorInfo ErrorInfo::ProgramEI(ErrLevel level, - const string &name, - const string &description, - const std::optional &hf) -{ - ErrorInfo ei(elError); - ei.name = name; - ei.description = description; - if (hf.has_value()) - ei.hint = std::optional(hf->str()); - else - ei.hint = std::nullopt; - return ei; -} - - - - - - -string showErrLine(const ErrLine &errLine) -{ - if (errLine.column > 0) { - return (format("(%1%:%2%)") % errLine.lineNumber % errLine.column).str(); + if (errPos.column > 0) { + return (format("(%1%:%2%)") % errPos.lineNumber % errPos.column).str(); } else { - return (format("(%1%)") % errLine.lineNumber).str(); + return (format("(%1%)") % errPos.lineNumber).str(); }; } -void printCodeLines(const string &prefix, const NixCode &nixCode) +void printCodeLines(const string &prefix, const ErrorInfo &einfo) { - - if (nixCode.errLine.has_value()) { + if (einfo.errPos.has_value()) { // previous line of code. - if (nixCode.errLine->prevLineOfCode.has_value()) { + if (einfo.prevLineOfCode.has_value()) { std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errLine->lineNumber - 1) - % *nixCode.errLine->prevLineOfCode + % prefix + % (einfo.errPos->lineNumber - 1) + % *einfo.prevLineOfCode << std::endl; } // line of code containing the error.%2$+5d% std::cout << format("%1% %|2$5d|| %3%") % prefix - % (nixCode.errLine->lineNumber) - % nixCode.errLine->errLineOfCode + % (einfo.errPos->lineNumber) + % einfo.errLineOfCode << std::endl; // error arrows for the column range. - if (nixCode.errLine->column > 0) { - int start = nixCode.errLine->column; + if (einfo.errPos->column > 0) { + int start = einfo.errPos->column; std::string spaces; for (int i = 0; i < start; ++i) { spaces.append(" "); @@ -89,11 +52,11 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) // next line of code. - if (nixCode.errLine->nextLineOfCode.has_value()) { + if (einfo.nextLineOfCode.has_value()) { std::cout << format("%1% %|2$5d|| %3%") % prefix - % (nixCode.errLine->lineNumber + 1) - % *nixCode.errLine->nextLineOfCode + % (einfo.errPos->lineNumber + 1) + % *einfo.nextLineOfCode << std::endl; } @@ -144,14 +107,14 @@ void printErrorInfo(const ErrorInfo &einfo) << std::endl; // filename. - if (einfo.nixCode.has_value()) { - if (einfo.nixCode->nixFile.has_value()) { - string eline = einfo.nixCode->errLine.has_value() - ? string(" ") + showErrLine(*einfo.nixCode->errLine) + if (einfo.errPos.has_value()) { + if (einfo.errPos->nixFile != "") { + string eline = einfo.errLineOfCode != "" + ? string(" ") + showErrPos(*einfo.errPos) : ""; std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % *einfo.nixCode->nixFile % eline << std::endl; + % prefix % einfo.errPos->nixFile % eline << std::endl; std::cout << prefix << std::endl; } else { std::cout << format("%1%from command line argument") % prefix << std::endl; @@ -164,8 +127,8 @@ void printErrorInfo(const ErrorInfo &einfo) std::cout << prefix << std::endl; // lines of code. - if (einfo.nixCode.has_value()) { - printCodeLines(prefix, *einfo.nixCode); + if (einfo.errLineOfCode != "") { + printCodeLines(prefix, einfo); std::cout << prefix << std::endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index e3bb2c1dd..b687bde81 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -17,23 +17,21 @@ typedef enum { elError } ErrLevel; -class ErrorInfo; - -class ErrLine +class ErrPos { public: int lineNumber; int column; - std::optional prevLineOfCode; - string errLineOfCode; - std::optional nextLineOfCode; -}; + string nixFile; -class NixCode -{ -public: - std::optional nixFile; - std::optional errLine; + template + ErrPos& operator=(const P &pos) + { + lineNumber = pos.line; + column = pos.column; + nixFile = pos.file.str(); + return *this; + } }; // ---------------------------------------------------------------- @@ -79,6 +77,11 @@ private: format fmt; }; +std::ostream& operator<<(std::ostream &os, const hintformat &hf) +{ + return os << hf.str(); +} + template inline hintformat hintfmt(const std::string & fs, const Args & ... args) { @@ -95,95 +98,56 @@ public: ErrLevel level; string name; string description; - std::optional nixCode; - std::optional hint; + std::optional hint; + std::optional prevLineOfCode; + string errLineOfCode; + std::optional nextLineOfCode; + std::optional errPos; static std::optional programName; - ErrorInfo& set_name(const string &name) { this->name = name; return *this; } - - static ErrorInfo ProgramError(const string &name, - const string &description, - const std::optional &hf); - - - static ErrorInfo ProgramWarning(const string &name, - const string &description, - const std::optional &hf); - - - template - static ErrorInfo NixLangError(const string &name, - const string &description, - const P &pos, - std::optional prevloc, - string loc, - std::optional nextloc, - const std::optional &hf) - { - return NixLangEI(elError, name, description, pos, prevloc, loc, nextloc, hf); - } - - - template - static ErrorInfo NixLangWarning(const string &name, - const string &description, - const P &pos, - std::optional prevloc, - string loc, - std::optional nextloc, - const std::optional &hf) - { - return NixLangEI(elWarning, name, description, pos, prevloc, loc, nextloc, hf); - } - - - private: - template - static ErrorInfo NixLangEI(ErrLevel level, - const string &name, - const string &description, - const P &pos, - std::optional prevloc, - string loc, - std::optional nextloc, - const std::optional &hf) - { - ErrorInfo ei(level); - ei.name = name; - ei.description = description; - if (hf.has_value()) - ei.hint = std::optional(hf->str()); - else - ei.hint = std::nullopt; + // template + // static ErrorInfo NixLangEI(ErrLevel level, + // const string &name, + // const string &description, + // const P &pos, + // std::optional prevloc, + // string loc, + // std::optional nextloc, + // const std::optional &hf) + // { + // ErrorInfo ei(level); + // ei.name = name; + // ei.description = description; + // if (hf.has_value()) + // ei.hint = std::optional(hf->str()); + // else + // ei.hint = std::nullopt; - ErrLine errline; - errline.lineNumber = pos.line; - errline.column = pos.column; - errline.prevLineOfCode = prevloc; - errline.errLineOfCode = loc; - errline.nextLineOfCode = nextloc; - NixCode nixcode; - nixcode.nixFile = pos.file; - nixcode.errLine = std::optional(errline); - ei.nixCode = std::optional(nixcode); + // ErrLine errline; + // errline.lineNumber = pos.line; + // errline.column = pos.column; + // errline.prevLineOfCode = prevloc; + // errline.errLineOfCode = loc; + // errline.nextLineOfCode = nextloc; + // NixCode nixcode; + // nixcode.nixFile = pos.file; + // nixcode.errLine = std::optional(errline); + // ei.nixCode = std::optional(nixcode); - return ei; - } + // return ei; + // } - static ErrorInfo ProgramEI(ErrLevel level, - const string &name, - const string &description, - const std::optional &hf); + // static ErrorInfo ProgramEI(ErrLevel level, + // const string &name, + // const string &description, + // const std::optional &hf); // constructor is protected, so only the builder classes can create an ErrorInfo. - ErrorInfo(ErrLevel level) - { - this->level = level; - } + }; // -------------------------------------------------------- From 54f91923c844a98c4f8fd4c06feab9421a879ad7 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 8 Apr 2020 09:48:21 -0600 Subject: [PATCH 098/350] return of NixCode --- src/error-demo/error-demo.cc | 47 +++++++++++-------- src/libutil/error.cc | 88 +++++++++++++++++------------------- src/libutil/error.hh | 63 +++++++------------------- 3 files changed, 85 insertions(+), 113 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 7eef0b162..f8ec95533 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -31,36 +31,43 @@ int main() .name = "name", .description = "error description", .hint = std::optional( - hintfmt("there was a %1%", "warning")) + hintfmt("there was a %1%", "warning")), }); // NixLangWarning adds nix file, line number, column range, and the lines of // code where a warning occurred. -/* SymbolTable testTable; - auto problem_symbol = testTable.create("problem"); + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); printErrorInfo( - ErrorInfo::NixLangWarning( - "warning name", - "warning description", - Pos(problem_symbol, 40, 13), - std::nullopt, - "this is the problem line of code", - std::nullopt, - hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + ErrorInfo{ + .level = elWarning, + .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); // NixLangError is just the same as NixLangWarning, except for the Error // flag. printErrorInfo( - ErrorInfo::NixLangError( - "error name", - "error description", - Pos(problem_symbol, 40, 13), - std::optional("previous line of code"), - "this is the problem line of code", - std::optional("next line of code"), - hintfmt("this hint has %1% templated %2%!!", "yellow", "values"))); + ErrorInfo{ + .level = elError, + .name = "error name", + .description = "error description", + .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::optional("previous line of code"), + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::optional("next line of code"), + }}); -*/ return 0; + + return 0; } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 24ed4df2e..1b9836059 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -17,51 +17,45 @@ string showErrPos(const ErrPos &errPos) }; } -void printCodeLines(const string &prefix, const ErrorInfo &einfo) +void printCodeLines(const string &prefix, const NixCode &nixCode) { - if (einfo.errPos.has_value()) { - // previous line of code. - if (einfo.prevLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (einfo.errPos->lineNumber - 1) - % *einfo.prevLineOfCode - << std::endl; - } - - // line of code containing the error.%2$+5d% + // previous line of code. + if (nixCode.prevLineOfCode.has_value()) { std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (einfo.errPos->lineNumber) - % einfo.errLineOfCode + % prefix + % (nixCode.errPos.lineNumber - 1) + % *nixCode.prevLineOfCode << std::endl; - - // error arrows for the column range. - if (einfo.errPos->column > 0) { - int start = einfo.errPos->column; - std::string spaces; - for (int i = 0; i < start; ++i) { - spaces.append(" "); - } - - std::string arrows("^"); - - std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; - } - - - - // next line of code. - if (einfo.nextLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (einfo.errPos->lineNumber + 1) - % *einfo.nextLineOfCode - << std::endl; - } - } + // line of code containing the error.%2$+5d% + std::cout << format("%1% %|2$5d|| %3%") + % prefix + % (nixCode.errPos.lineNumber) + % nixCode.errLineOfCode + << std::endl; + + // error arrows for the column range. + if (nixCode.errPos.column > 0) { + int start = nixCode.errPos.column; + std::string spaces; + for (int i = 0; i < start; ++i) { + spaces.append(" "); + } + + std::string arrows("^"); + + std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; + } + + // next line of code. + if (nixCode.nextLineOfCode.has_value()) { + std::cout << format("%1% %|2$5d|| %3%") + % prefix + % (nixCode.errPos.lineNumber + 1) + % *nixCode.nextLineOfCode + << std::endl; + } } void printErrorInfo(const ErrorInfo &einfo) @@ -107,14 +101,14 @@ void printErrorInfo(const ErrorInfo &einfo) << std::endl; // filename. - if (einfo.errPos.has_value()) { - if (einfo.errPos->nixFile != "") { - string eline = einfo.errLineOfCode != "" - ? string(" ") + showErrPos(*einfo.errPos) + if (einfo.nixCode.has_value()) { + if (einfo.nixCode->errPos.nixFile != "") { + string eline = einfo.nixCode->errLineOfCode != "" + ? string(" ") + showErrPos(einfo.nixCode->errPos) : ""; std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % einfo.errPos->nixFile % eline << std::endl; + % prefix % einfo.nixCode->errPos.nixFile % eline << std::endl; std::cout << prefix << std::endl; } else { std::cout << format("%1%from command line argument") % prefix << std::endl; @@ -127,8 +121,8 @@ void printErrorInfo(const ErrorInfo &einfo) std::cout << prefix << std::endl; // lines of code. - if (einfo.errLineOfCode != "") { - printCodeLines(prefix, einfo); + if (einfo.nixCode->errLineOfCode != "") { + printCodeLines(prefix, *einfo.nixCode); std::cout << prefix << std::endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index b687bde81..7e76c0079 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -29,9 +29,24 @@ public: { lineNumber = pos.line; column = pos.column; - nixFile = pos.file.str(); + nixFile = pos.file; return *this; } + + template + ErrPos(const P &p) + { + *this = p; + } +}; + +class NixCode +{ +public: + ErrPos errPos; + std::optional prevLineOfCode; + string errLineOfCode; + std::optional nextLineOfCode; }; // ---------------------------------------------------------------- @@ -99,55 +114,11 @@ public: string name; string description; std::optional hint; - std::optional prevLineOfCode; - string errLineOfCode; - std::optional nextLineOfCode; - std::optional errPos; + std::optional nixCode; static std::optional programName; private: - // template - // static ErrorInfo NixLangEI(ErrLevel level, - // const string &name, - // const string &description, - // const P &pos, - // std::optional prevloc, - // string loc, - // std::optional nextloc, - // const std::optional &hf) - // { - // ErrorInfo ei(level); - // ei.name = name; - // ei.description = description; - // if (hf.has_value()) - // ei.hint = std::optional(hf->str()); - // else - // ei.hint = std::nullopt; - - // ErrLine errline; - // errline.lineNumber = pos.line; - // errline.column = pos.column; - // errline.prevLineOfCode = prevloc; - // errline.errLineOfCode = loc; - // errline.nextLineOfCode = nextloc; - // NixCode nixcode; - // nixcode.nixFile = pos.file; - // nixcode.errLine = std::optional(errline); - // ei.nixCode = std::optional(nixcode); - - // return ei; - // } - - // static ErrorInfo ProgramEI(ErrLevel level, - // const string &name, - // const string &description, - // const std::optional &hf); - - - - // constructor is protected, so only the builder classes can create an ErrorInfo. - }; // -------------------------------------------------------- From 555baa8fb07d4296c271ebc263a0a8fbe30268ee Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 8 Apr 2020 09:56:10 -0600 Subject: [PATCH 099/350] comments --- src/error-demo/error-demo.cc | 17 +++++------------ src/libutil/error.hh | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index f8ec95533..a9ff6057c 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -11,21 +11,15 @@ int main() // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); - // There are currently four constructor functions: - // - // ProgramError, ProgramWarning, NixLangError, NixLangWarning. - // - - // ProgramError takes name, description, and an optional hint. + // Error in a program; no hint and no nix code. printErrorInfo( ErrorInfo { .level = elError, .name = "name", .description = "error description", }); - // ProgramWarning takes name, description, and an optional hint. - // The hint is in the form of a hintfmt class, which wraps boost::format(), - // and makes all the substituted text yellow. + // Warning with name, description, and hint. + // The hintfmt function makes all the substituted text yellow. printErrorInfo( ErrorInfo { .level = elWarning, .name = "name", @@ -35,7 +29,7 @@ int main() }); - // NixLangWarning adds nix file, line number, column range, and the lines of + // Warning with nix file, line number, column, and the lines of // code where a warning occurred. SymbolTable testTable; auto problem_file = testTable.create("myfile.nix"); @@ -53,8 +47,7 @@ int main() .nextLineOfCode = std::nullopt }}); - // NixLangError is just the same as NixLangWarning, except for the Error - // flag. + // Error with previous and next lines of code. printErrorInfo( ErrorInfo{ .level = elError, diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7e76c0079..8286eaab1 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -50,7 +50,7 @@ public: }; // ---------------------------------------------------------------- -// format for hints. same as fmt, except templated values +// format function for hints. same as fmt, except templated values // are always in yellow. template From bf81b3155998152582abdad1550277cf8ca32e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 8 Apr 2020 18:27:10 +0200 Subject: [PATCH 100/350] build.cc: improve message if home directory exists --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 527d7ac42..632982ecd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2161,7 +2161,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); + throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete From 8c2bf15c4fbf5cd18d3ee0b88f6b510e8881a622 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 8 Apr 2020 11:17:02 -0600 Subject: [PATCH 101/350] format -> fmt --- src/libutil/error.cc | 57 +++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1b9836059..138c50d2b 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,9 +11,9 @@ std::optional ErrorInfo::programName = std::nullopt; string showErrPos(const ErrPos &errPos) { if (errPos.column > 0) { - return (format("(%1%:%2%)") % errPos.lineNumber % errPos.column).str(); + return fmt("(%1%:%2%)", errPos.lineNumber, errPos.column); } else { - return (format("(%1%)") % errPos.lineNumber).str(); + return fmt("(%1%)", errPos.lineNumber); }; } @@ -21,18 +21,18 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) { // previous line of code. if (nixCode.prevLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errPos.lineNumber - 1) - % *nixCode.prevLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.lineNumber - 1), + *nixCode.prevLineOfCode) << std::endl; } // line of code containing the error.%2$+5d% - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errPos.lineNumber) - % nixCode.errLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.lineNumber), + nixCode.errLineOfCode) << std::endl; // error arrows for the column range. @@ -45,15 +45,18 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) std::string arrows("^"); - std::cout << format("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL) % prefix % spaces % arrows << std::endl; + std::cout << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows) << std::endl; } // next line of code. if (nixCode.nextLineOfCode.has_value()) { - std::cout << format("%1% %|2$5d|| %3%") - % prefix - % (nixCode.errPos.lineNumber + 1) - % *nixCode.nextLineOfCode + std::cout << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.lineNumber + 1), + *nixCode.nextLineOfCode) << std::endl; } } @@ -78,8 +81,8 @@ void printErrorInfo(const ErrorInfo &einfo) break; } default: { - levelString = (format("invalid error level: %1%") % einfo.level).str(); - break; + levelString = fmt("invalid error level: %1%", einfo.level); + break; } } @@ -91,13 +94,13 @@ void printErrorInfo(const ErrorInfo &einfo) dashes.append("-"); // divider. - std::cout << format("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL) - % prefix - % levelString - % "---" - % einfo.name - % dashes - % einfo.programName.value_or("") + std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL, + prefix, + levelString, + "---", + einfo.name, + dashes, + einfo.programName.value_or("")) << std::endl; // filename. @@ -107,11 +110,11 @@ void printErrorInfo(const ErrorInfo &einfo) ? string(" ") + showErrPos(einfo.nixCode->errPos) : ""; - std::cout << format("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL) - % prefix % einfo.nixCode->errPos.nixFile % eline << std::endl; + std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL + , prefix, einfo.nixCode->errPos.nixFile, eline) << std::endl; std::cout << prefix << std::endl; } else { - std::cout << format("%1%from command line argument") % prefix << std::endl; + std::cout << fmt("%1%from command line argument", prefix) << std::endl; std::cout << prefix << std::endl; } } From 805ffe1bc93b14ad8d1132c20179e85a914c91e6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 8 Apr 2020 11:33:46 -0600 Subject: [PATCH 102/350] indention --- src/libutil/error.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 138c50d2b..665361a2a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -82,7 +82,7 @@ void printErrorInfo(const ErrorInfo &einfo) } default: { levelString = fmt("invalid error level: %1%", einfo.level); - break; + break; } } From 65ef57e0cbded52652510355304303a7b71586af Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 13:30:45 +0200 Subject: [PATCH 103/350] DownloadRequest -> DataTransferRequest --- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/download.cc | 13 ++++++------- src/libstore/download.hh | 12 ++++++------ src/libstore/http-binary-cache-store.cc | 8 ++++---- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..a939f040f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -36,7 +36,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DownloadRequest request(url); + DataTransferRequest request(url); request.verifyTLS = false; request.decompress = false; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index bc875f653..77f3c527a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -49,7 +49,7 @@ struct CurlDownloader : public Downloader struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; - DownloadRequest request; + DataTransferRequest request; DownloadResult result; Activity act; bool done = false; // whether either the success or failure function has been called @@ -73,7 +73,7 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, + const DataTransferRequest & request, Callback && callback) : downloader(downloader) , request(request) @@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DownloadRequest & request, + void enqueueDownload(const DataTransferRequest & request, Callback callback) override { /* Ugly hack to support s3:// URIs. */ @@ -687,7 +687,7 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DownloadRequest & request) +std::future Downloader::enqueueDownload(const DataTransferRequest & request) { auto promise = std::make_shared>(); enqueueDownload(request, @@ -701,12 +701,12 @@ std::future Downloader::enqueueDownload(const DownloadRequest & return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) +DownloadResult Downloader::download(const DataTransferRequest & request) { return enqueueDownload(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) +void Downloader::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because that would cause the sink to execute on the downloader @@ -801,7 +801,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } - bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 28c8a9162..89d3fce7f 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -33,7 +33,7 @@ struct DownloadSettings : Config extern DownloadSettings downloadSettings; -struct DownloadRequest +struct DataTransferRequest { std::string uri; std::string expectedETag; @@ -47,7 +47,7 @@ struct DownloadRequest std::string mimeType; std::function dataCallback; - DownloadRequest(const std::string & uri) + DataTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -74,17 +74,17 @@ struct Downloader /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ - virtual void enqueueDownload(const DownloadRequest & request, + virtual void enqueueDownload(const DataTransferRequest & request, Callback callback) = 0; - std::future enqueueDownload(const DownloadRequest & request); + std::future enqueueDownload(const DataTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + DownloadResult download(const DataTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); + void download(DataTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 011794c62..e152e7524 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,7 +85,7 @@ protected: checkEnabled(); try { - DownloadRequest request(cacheUri + "/" + path); + DataTransferRequest request(cacheUri + "/" + path); request.head = true; getDownloader()->download(request); return true; @@ -103,7 +103,7 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DownloadRequest(cacheUri + "/" + path); + auto req = DataTransferRequest(cacheUri + "/" + path); req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { @@ -113,9 +113,9 @@ protected: } } - DownloadRequest makeRequest(const std::string & path) + DataTransferRequest makeRequest(const std::string & path) { - DownloadRequest request(cacheUri + "/" + path); + DataTransferRequest request(cacheUri + "/" + path); return request; } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 2b9254659..bfd65da98 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -180,7 +180,7 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DownloadRequest req(actualUri); + DataTransferRequest req(actualUri); req.decompress = false; getDownloader()->download(std::move(req), sink); } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c05c29517..aeaf7b09c 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -138,7 +138,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DownloadRequest(storePathsUrl); + auto req = DataTransferRequest(storePathsUrl); auto res = getDownloader()->download(req); auto state = std::make_unique(Strings(), store); From 741e9012d3fa6f0fb41a4a5662c6b8b38ecfaa1f Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:10:58 +0200 Subject: [PATCH 104/350] Rename src/lib/download.* to src/lib/datatransfer.* --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/parser.y | 2 +- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/{download.cc => datatransfer.cc} | 2 +- src/libstore/{download.hh => datatransfer.hh} | 0 src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/main.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename src/libstore/{download.cc => datatransfer.cc} (99%) rename src/libstore/{download.hh => datatransfer.hh} (100%) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 82bfeac36..26460601d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,6 +1,6 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "download.hh" +#include "datatransfer.hh" #include "util.hh" #include "eval.hh" #include "fetchers.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..7a20c6fa6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "download.hh" +#include "datatransfer.hh" #include "json.hh" #include "function-trace.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6f25f5cf0..96b6f0ecd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -544,7 +544,7 @@ formal #include #include "eval.hh" -#include "download.hh" +#include "datatransfer.hh" #include "fetchers.hh" #include "store-api.hh" diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 632982ecd..db01d9510 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "download.hh" +#include "datatransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index a939f040f..01d21fa98 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" diff --git a/src/libstore/download.cc b/src/libstore/datatransfer.cc similarity index 99% rename from src/libstore/download.cc rename to src/libstore/datatransfer.cc index 77f3c527a..30ad48577 100644 --- a/src/libstore/download.cc +++ b/src/libstore/datatransfer.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "datatransfer.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/download.hh b/src/libstore/datatransfer.hh similarity index 100% rename from src/libstore/download.hh rename to src/libstore/datatransfer.hh diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e152e7524..1fccee518 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "datatransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..ed3c690b6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "download.hh" +#include "datatransfer.hh" #include "istringstream_nocopy.hh" #include diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 2a9defb4e..d0719194d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "globals.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "../nix/legacy.hh" #include "fetchers.hh" diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index bfd65da98..58a180e72 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,6 +1,6 @@ #include "hash.hh" #include "shared.hh" -#include "download.hh" +#include "datatransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 3b5f5516f..ad4102201 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "download.hh" +#include "datatransfer.hh" #include "finally.hh" #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index aeaf7b09c..414689ec5 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "download.hh" +#include "datatransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" From e5cc53beec62a36ebdc219e26b87f501ddb84a4f Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:14:49 +0200 Subject: [PATCH 105/350] DownloadSettings -> DataTransferSettings --- src/libstore/datatransfer.cc | 14 +++++++------- src/libstore/datatransfer.hh | 6 +++--- src/nix/main.cc | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 30ad48577..62f546cfa 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -27,9 +27,9 @@ using namespace std::string_literals; namespace nix { -DownloadSettings downloadSettings; +DataTransferSettings dataTransferSettings; -static GlobalConfig::Register r1(&downloadSettings); +static GlobalConfig::Register r1(&dataTransferSettings); std::string resolveUri(const std::string & uri) { @@ -257,12 +257,12 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) + if (dataTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -297,10 +297,10 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); + dataTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 89d3fce7f..16955ff06 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DownloadSettings : Config +struct DataTransferSettings : Config { Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,7 +31,7 @@ struct DownloadSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DownloadSettings downloadSettings; +extern DataTransferSettings dataTransferSettings; struct DataTransferRequest { @@ -39,7 +39,7 @@ struct DataTransferRequest std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = downloadSettings.tries; + size_t tries = dataTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/nix/main.cc b/src/nix/main.cc index ad4102201..6bb04d8b6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits::max(); - if (!downloadSettings.tries.overriden) - downloadSettings.tries = 0; - if (!downloadSettings.connectTimeout.overriden) - downloadSettings.connectTimeout = 1; + if (!dataTransferSettings.tries.overriden) + dataTransferSettings.tries = 0; + if (!dataTransferSettings.connectTimeout.overriden) + dataTransferSettings.connectTimeout = 1; } if (args.refresh) From 142ed7fe45a2b1d566e4401b3974cb29b8944fc6 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:18:51 +0200 Subject: [PATCH 106/350] DownloadResult -> DataTransferResult --- src/libstore/datatransfer.cc | 20 ++++++++++---------- src/libstore/datatransfer.hh | 8 ++++---- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 4 ++-- src/libstore/s3.hh | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 62f546cfa..06aa86d97 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -50,10 +50,10 @@ struct CurlDownloader : public Downloader { CurlDownloader & downloader; DataTransferRequest request; - DownloadResult result; + DataTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback callback; + Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -74,7 +74,7 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DataTransferRequest & request, - Callback && callback) + Callback && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, @@ -642,7 +642,7 @@ struct CurlDownloader : public Downloader #endif void enqueueDownload(const DataTransferRequest & request, - Callback callback) override + Callback callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,7 +660,7 @@ struct CurlDownloader : public Downloader // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; + DataTransferResult res; if (!s3Res.data) throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; @@ -687,11 +687,11 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DataTransferRequest & request) +std::future Downloader::enqueueDownload(const DataTransferRequest & request) { - auto promise = std::make_shared>(); + auto promise = std::make_shared>(); enqueueDownload(request, - {[promise](std::future fut) { + {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,7 +701,7 @@ std::future Downloader::enqueueDownload(const DataTransferReques return promise->get_future(); } -DownloadResult Downloader::download(const DataTransferRequest & request) +DataTransferResult Downloader::download(const DataTransferRequest & request) { return enqueueDownload(request).get(); } @@ -756,7 +756,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink) }; enqueueDownload(request, - {[_state](std::future fut) { + {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; try { diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 16955ff06..b9e55655f 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -56,7 +56,7 @@ struct DataTransferRequest } }; -struct DownloadResult +struct DataTransferResult { bool cached = false; std::string etag; @@ -75,12 +75,12 @@ struct Downloader the download. The future may throw a DownloadError exception. */ virtual void enqueueDownload(const DataTransferRequest & request, - Callback callback) = 0; + Callback callback) = 0; - std::future enqueueDownload(const DataTransferRequest & request); + std::future enqueueDownload(const DataTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DataTransferRequest & request); + DataTransferResult download(const DataTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1fccee518..24a98d073 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -143,7 +143,7 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future result) { + {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); } catch (DownloadError & e) { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ed3c690b6..478cb9f84 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -132,7 +132,7 @@ ref S3Helper::makeConfig(const string & region return res; } -S3Helper::DownloadResult S3Helper::getObject( +S3Helper::DataTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); - DownloadResult res; + DataTransferResult res; auto now1 = std::chrono::steady_clock::now(); diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index ef5f23d0f..d7d309243 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DownloadResult + struct DataTransferResult { std::shared_ptr data; unsigned int durationMs; }; - DownloadResult getObject( + DataTransferResult getObject( const std::string & bucketName, const std::string & key); }; From 2df2741ec6e9411739b2049d7487b7b8a26d4547 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 15:37:33 +0200 Subject: [PATCH 107/350] enqueueDownload -> enqueueDataTransfer --- src/libstore/datatransfer.cc | 10 +++++----- src/libstore/datatransfer.hh | 4 ++-- src/libstore/http-binary-cache-store.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 06aa86d97..9475c24ec 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -641,7 +641,7 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DataTransferRequest & request, + void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) override { /* Ugly hack to support s3:// URIs. */ @@ -687,10 +687,10 @@ ref makeDownloader() return make_ref(); } -std::future Downloader::enqueueDownload(const DataTransferRequest & request) +std::future Downloader::enqueueDataTransfer(const DataTransferRequest & request) { auto promise = std::make_shared>(); - enqueueDownload(request, + enqueueDataTransfer(request, {[promise](std::future fut) { try { promise->set_value(fut.get()); @@ -703,7 +703,7 @@ std::future Downloader::enqueueDownload(const DataTransferRe DataTransferResult Downloader::download(const DataTransferRequest & request) { - return enqueueDownload(request).get(); + return enqueueDataTransfer(request).get(); } void Downloader::download(DataTransferRequest && request, Sink & sink) @@ -755,7 +755,7 @@ void Downloader::download(DataTransferRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDownload(request, + enqueueDataTransfer(request, {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index b9e55655f..53159bdb4 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -74,10 +74,10 @@ struct Downloader /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ - virtual void enqueueDownload(const DataTransferRequest & request, + virtual void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) = 0; - std::future enqueueDownload(const DataTransferRequest & request); + std::future enqueueDataTransfer(const DataTransferRequest & request); /* Synchronously download a file. */ DataTransferResult download(const DataTransferRequest & request); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 24a98d073..861287a33 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -142,7 +142,7 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDownloader()->enqueueDownload(request, + getDownloader()->enqueueDataTransfer(request, {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); From cd391206e6d29608bc3e25301b9921f1ac189086 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 22:57:09 +0200 Subject: [PATCH 108/350] {get,make,new}Downloader -> DataTransfer --- src/libstore/builtins/fetchurl.cc | 6 ++-- src/libstore/datatransfer.cc | 36 ++++++++++++------------ src/libstore/datatransfer.hh | 18 ++++++------ src/libstore/http-binary-cache-store.cc | 14 ++++----- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 01d21fa98..0b6db8f0c 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh downloader here because we're in + /* Note: have to use a fresh dataTransfer here because we're in a forked process. */ - auto downloader = makeDownloader(); + auto dataTransfer = makeDataTransfer(); auto fetch = [&](const std::string & url) { @@ -42,7 +42,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); + dataTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 9475c24ec..6ea9f9324 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri) return uri; } -struct CurlDownloader : public Downloader +struct curlDataTransfer : public DataTransfer { CURLM * curlm = 0; @@ -48,7 +48,7 @@ struct CurlDownloader : public Downloader struct DownloadItem : public std::enable_shared_from_this { - CurlDownloader & downloader; + curlDataTransfer & dataTransfer; DataTransferRequest request; DataTransferResult result; Activity act; @@ -72,10 +72,10 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; - DownloadItem(CurlDownloader & downloader, + DownloadItem(curlDataTransfer & dataTransfer, const DataTransferRequest & request, Callback && callback) - : downloader(downloader) + : dataTransfer(dataTransfer) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), @@ -106,7 +106,7 @@ struct CurlDownloader : public Downloader { if (req) { if (active) - curl_multi_remove_handle(downloader.curlm, req); + curl_multi_remove_handle(dataTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); @@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); + dataTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; - CurlDownloader() + curlDataTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~CurlDownloader() + ~curlDataTransfer() { stopWorkerThread(); @@ -676,18 +676,18 @@ struct CurlDownloader : public Downloader } }; -ref getDownloader() +ref getDataTransfer() { - static ref downloader = makeDownloader(); - return downloader; + static ref dataTransfer = makeDataTransfer(); + return dataTransfer; } -ref makeDownloader() +ref makeDataTransfer() { - return make_ref(); + return make_ref(); } -std::future Downloader::enqueueDataTransfer(const DataTransferRequest & request) +std::future DataTransfer::enqueueDataTransfer(const DataTransferRequest & request) { auto promise = std::make_shared>(); enqueueDataTransfer(request, @@ -701,15 +701,15 @@ std::future Downloader::enqueueDataTransfer(const DataTransf return promise->get_future(); } -DataTransferResult Downloader::download(const DataTransferRequest & request) +DataTransferResult DataTransfer::download(const DataTransferRequest & request) { return enqueueDataTransfer(request).get(); } -void Downloader::download(DataTransferRequest && request, Sink & sink) +void DataTransfer::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader + that would cause the sink to execute on the dataTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 53159bdb4..e1c5196e0 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -67,11 +67,11 @@ struct DataTransferResult class Store; -struct Downloader +struct DataTransfer { - virtual ~Downloader() { } + virtual ~DataTransfer() { } - /* Enqueue a download request, returning a future to the result of + /* Enqueue a data transfer request, returning a future to the result of the download. The future may throw a DownloadError exception. */ virtual void enqueueDataTransfer(const DataTransferRequest & request, @@ -89,18 +89,18 @@ struct Downloader enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared Downloader object. Using this object is preferred +/* Return a shared DataTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDownloader(); +ref getDataTransfer(); -/* Return a new Downloader object. */ -ref makeDownloader(); +/* Return a new DataTransfer object. */ +ref makeDataTransfer(); class DownloadError : public Error { public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) + DataTransfer::Error error; + DownloadError(DataTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 861287a33..9a19af5e0 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -87,12 +87,12 @@ protected: try { DataTransferRequest request(cacheUri + "/" + path); request.head = true; - getDownloader()->download(request); + getDataTransfer()->download(request); return true; } catch (DownloadError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return false; maybeDisable(); throw; @@ -107,7 +107,7 @@ protected: req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDownloader()->download(req); + getDataTransfer()->download(req); } catch (DownloadError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDownloader()->download(std::move(request), sink); + getDataTransfer()->download(std::move(request), sink); } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDownloader()->enqueueDataTransfer(request, + getDataTransfer()->enqueueDataTransfer(request, {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); callbackPtr->rethrow(); diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 58a180e72..4c81e7d82 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -182,7 +182,7 @@ static int _main(int argc, char * * argv) DataTransferRequest req(actualUri); req.decompress = false; - getDownloader()->download(std::move(req), sink); + getDataTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 414689ec5..a4eda90b4 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -139,7 +139,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand // FIXME: use nixos.org? auto req = DataTransferRequest(storePathsUrl); - auto res = getDownloader()->download(req); + auto res = getDataTransfer()->download(req); auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); From 213d124277460b316cfa03d43e4ec279bba86219 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 22:59:36 +0200 Subject: [PATCH 109/350] DownloadItem -> TransferItem --- src/libstore/datatransfer.cc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 6ea9f9324..452af107d 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -46,7 +46,7 @@ struct curlDataTransfer : public DataTransfer std::random_device rd; std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this + struct TransferItem : public std::enable_shared_from_this { curlDataTransfer & dataTransfer; DataTransferRequest request; @@ -72,7 +72,7 @@ struct curlDataTransfer : public DataTransfer curl_off_t writtenToSink = 0; - DownloadItem(curlDataTransfer & dataTransfer, + TransferItem(curlDataTransfer & dataTransfer, const DataTransferRequest & request, Callback && callback) : dataTransfer(dataTransfer) @@ -102,7 +102,7 @@ struct curlDataTransfer : public DataTransfer requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } - ~DownloadItem() + ~TransferItem() { if (req) { if (active) @@ -156,7 +156,7 @@ struct curlDataTransfer : public DataTransfer static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) @@ -198,7 +198,7 @@ struct curlDataTransfer : public DataTransfer static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -213,7 +213,7 @@ struct curlDataTransfer : public DataTransfer static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -237,7 +237,7 @@ struct curlDataTransfer : public DataTransfer static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } void init() @@ -248,7 +248,7 @@ struct curlDataTransfer : public DataTransfer if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); @@ -267,9 +267,9 @@ struct curlDataTransfer : public DataTransfer else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); @@ -439,12 +439,12 @@ struct curlDataTransfer : public DataTransfer struct State { struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { return i1->embargo > i2->embargo; } }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::priority_queue, std::vector>, EmbargoComparator> incoming; }; Sync state_; @@ -504,7 +504,7 @@ struct curlDataTransfer : public DataTransfer stopWorkerThread(); }); - std::map> items; + std::map> items; bool quit = false; @@ -561,7 +561,7 @@ struct curlDataTransfer : public DataTransfer throw SysError("reading curl wakeup socket"); } - std::vector> incoming; + std::vector> incoming; auto now = std::chrono::steady_clock::now(); { @@ -609,7 +609,7 @@ struct curlDataTransfer : public DataTransfer } } - void enqueueItem(std::shared_ptr item) + void enqueueItem(std::shared_ptr item) { if (item->request.data && !hasPrefix(item->request.uri, "http://") @@ -672,7 +672,7 @@ struct curlDataTransfer : public DataTransfer return; } - enqueueItem(std::make_shared(*this, request, std::move(callback))); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; From c4c1ae0a00e8dd3258af58c15b664828b592133e Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:00:43 +0200 Subject: [PATCH 110/350] DownloadError -> DataTransferError --- src/libexpr/parser.y | 2 +- src/libstore/datatransfer.cc | 10 +++++----- src/libstore/datatransfer.hh | 6 +++--- src/libstore/http-binary-cache-store.cc | 8 ++++---- src/nix-channel/nix-channel.cc | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 96b6f0ecd..8fbe59301 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -690,7 +690,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl try { res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; - } catch (DownloadError & e) { + } catch (DataTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; } diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 452af107d..be5dd27cd 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -112,7 +112,7 @@ struct curlDataTransfer : public DataTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); } catch (...) { ignoreException(); } @@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, + ? DataTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DownloadError(err, + : DataTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -662,7 +662,7 @@ struct curlDataTransfer : public DataTransfer auto s3Res = s3Helper.getObject(bucketName, key); DataTransferResult res; if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index e1c5196e0..f2ea53ae3 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -72,7 +72,7 @@ struct DataTransfer virtual ~DataTransfer() { } /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a DownloadError + the download. The future may throw a DataTransferError exception. */ virtual void enqueueDataTransfer(const DataTransferRequest & request, Callback callback) = 0; @@ -96,11 +96,11 @@ ref getDataTransfer(); /* Return a new DataTransfer object. */ ref makeDataTransfer(); -class DownloadError : public Error +class DataTransferError : public Error { public: DataTransfer::Error error; - DownloadError(DataTransfer::Error error, const FormatOrString & fs) + DataTransferError(DataTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9a19af5e0..5e446ddf2 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -89,7 +89,7 @@ protected: request.head = true; getDataTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (DataTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) @@ -108,7 +108,7 @@ protected: req.mimeType = mimeType; try { getDataTransfer()->download(req); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } @@ -125,7 +125,7 @@ protected: auto request(makeRequest(path)); try { getDataTransfer()->download(std::move(request), sink); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); @@ -146,7 +146,7 @@ protected: {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index d0719194d..0f8024d7b 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,7 @@ static void update(const StringSet & channelNames) // Download the channel tarball. try { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (DownloadError & e) { + } catch (DataTransferError & e) { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } From a0c5931208042da39bb6a5e80a4b27cf50f665d6 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:08:39 +0200 Subject: [PATCH 111/350] actDownload -> actDataTransfer --- src/libstore/build.cc | 2 +- src/libstore/datatransfer.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/logging.hh | 2 +- src/nix/progress-bar.cc | 6 +++--- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index db01d9510..f091a000d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -361,7 +361,7 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index be5dd27cd..9ed06b5c1 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -77,7 +77,7 @@ struct curlDataTransfer : public DataTransfer Callback && callback) : dataTransfer(dataTransfer) , request(request) - , act(*logger, lvlTalkative, actDownload, + , act(*logger, lvlTalkative, actDataTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index fa5c84a27..001a3fb28 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDownload) + if (trusted || type == actDataTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index beb5e6b64..423ed95da 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -17,7 +17,7 @@ typedef enum { typedef enum { actUnknown = 0, actCopyPath = 100, - actDownload = 101, + actDataTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 26631416c..d108896ad 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -190,8 +190,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) - || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +416,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); From 7848372b0f683a5a4db5f86fd998e8df5fa22715 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:34:31 +0200 Subject: [PATCH 112/350] Add upload method --- src/libstore/datatransfer.cc | 6 ++++++ src/libstore/datatransfer.hh | 3 +++ src/libstore/http-binary-cache-store.cc | 3 +-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index 9ed06b5c1..dbdf07ad4 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -706,6 +706,12 @@ DataTransferResult DataTransfer::download(const DataTransferRequest & request) return enqueueDataTransfer(request).get(); } +DataTransferResult DataTransfer::upload(const DataTransferRequest & request) +{ + /* Note: this method is the same as download, but helps in readability */ + return enqueueDataTransfer(request).get(); +} + void DataTransfer::download(DataTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index f2ea53ae3..68d97ceb1 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -82,6 +82,9 @@ struct DataTransfer /* Synchronously download a file. */ DataTransferResult download(const DataTransferRequest & request); + /* Synchronously upload a file. */ + DataTransferResult upload(const DataTransferRequest & request); + /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ void download(DataTransferRequest && request, Sink & sink); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5e446ddf2..c75846bc3 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -107,7 +107,7 @@ protected: req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDataTransfer()->download(req); + getDataTransfer()->upload(req); } catch (DataTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } @@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([]( }); } - From c330109bfa38370f7cef6449efb88309a78e1684 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:43:43 +0200 Subject: [PATCH 113/350] DataTransfer -> FileTransfer --- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 8 +- src/libstore/datatransfer.cc | 94 ++++++++++++------------ src/libstore/datatransfer.hh | 44 +++++------ src/libstore/http-binary-cache-store.cc | 32 ++++---- src/libstore/s3-binary-cache-store.cc | 4 +- src/libstore/s3.hh | 4 +- src/libutil/logging.cc | 2 +- src/libutil/logging.hh | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 4 +- src/nix/main.cc | 8 +- src/nix/progress-bar.cc | 6 +- src/nix/upgrade-nix.cc | 4 +- 13 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f091a000d..9baa2cf1a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -361,7 +361,7 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDataTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 0b6db8f0c..1432de9d7 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh dataTransfer here because we're in + /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ - auto dataTransfer = makeDataTransfer(); + auto fileTransfer = makeFileTransfer(); auto fetch = [&](const std::string & url) { @@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DataTransferRequest request(url); + FileTransferRequest request(url); request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - dataTransfer->download(std::move(request), *decompressor); + fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); diff --git a/src/libstore/datatransfer.cc b/src/libstore/datatransfer.cc index dbdf07ad4..5cc63a1d8 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/datatransfer.cc @@ -27,9 +27,9 @@ using namespace std::string_literals; namespace nix { -DataTransferSettings dataTransferSettings; +FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&dataTransferSettings); +static GlobalConfig::Register r1(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -39,7 +39,7 @@ std::string resolveUri(const std::string & uri) return uri; } -struct curlDataTransfer : public DataTransfer +struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; @@ -48,12 +48,12 @@ struct curlDataTransfer : public DataTransfer struct TransferItem : public std::enable_shared_from_this { - curlDataTransfer & dataTransfer; - DataTransferRequest request; - DataTransferResult result; + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback callback; + Callback callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -72,12 +72,12 @@ struct curlDataTransfer : public DataTransfer curl_off_t writtenToSink = 0; - TransferItem(curlDataTransfer & dataTransfer, - const DataTransferRequest & request, - Callback && callback) - : dataTransfer(dataTransfer) + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback && callback) + : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actDataTransfer, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) @@ -106,13 +106,13 @@ struct curlDataTransfer : public DataTransfer { if (req) { if (active) - curl_multi_remove_handle(dataTransfer.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DataTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); } catch (...) { ignoreException(); } @@ -257,12 +257,12 @@ struct curlDataTransfer : public DataTransfer curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (dataTransferSettings.userAgentSuffix != "" ? " " + dataTransferSettings.userAgentSuffix.get() : "")).c_str()); + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (dataTransferSettings.enableHttp2) + if (fileTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); @@ -297,10 +297,10 @@ struct curlDataTransfer : public DataTransfer curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, dataTransferSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, dataTransferSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -401,14 +401,14 @@ struct curlDataTransfer : public DataTransfer auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DataTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DataTransferError(err, + ? FileTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DataTransferError(err, + : FileTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -422,13 +422,13 @@ struct curlDataTransfer : public DataTransfer || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(dataTransfer.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - dataTransfer.enqueueItem(shared_from_this()); + fileTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -456,7 +456,7 @@ struct curlDataTransfer : public DataTransfer std::thread workerThread; - curlDataTransfer() + curlFileTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -469,7 +469,7 @@ struct curlDataTransfer : public DataTransfer #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - dataTransferSettings.httpConnections.get()); + fileTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); @@ -478,7 +478,7 @@ struct curlDataTransfer : public DataTransfer workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~curlDataTransfer() + ~curlFileTransfer() { stopWorkerThread(); @@ -641,8 +641,8 @@ struct curlDataTransfer : public DataTransfer } #endif - void enqueueDataTransfer(const DataTransferRequest & request, - Callback callback) override + void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,9 +660,9 @@ struct curlDataTransfer : public DataTransfer // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DataTransferResult res; + FileTransferResult res; if (!s3Res.data) - throw DataTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else @@ -676,22 +676,22 @@ struct curlDataTransfer : public DataTransfer } }; -ref getDataTransfer() +ref getFileTransfer() { - static ref dataTransfer = makeDataTransfer(); - return dataTransfer; + static ref fileTransfer = makeFileTransfer(); + return fileTransfer; } -ref makeDataTransfer() +ref makeFileTransfer() { - return make_ref(); + return make_ref(); } -std::future DataTransfer::enqueueDataTransfer(const DataTransferRequest & request) +std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { - auto promise = std::make_shared>(); - enqueueDataTransfer(request, - {[promise](std::future fut) { + auto promise = std::make_shared>(); + enqueueFileTransfer(request, + {[promise](std::future fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,21 +701,21 @@ std::future DataTransfer::enqueueDataTransfer(const DataTran return promise->get_future(); } -DataTransferResult DataTransfer::download(const DataTransferRequest & request) +FileTransferResult FileTransfer::download(const FileTransferRequest & request) { - return enqueueDataTransfer(request).get(); + return enqueueFileTransfer(request).get(); } -DataTransferResult DataTransfer::upload(const DataTransferRequest & request) +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) { /* Note: this method is the same as download, but helps in readability */ - return enqueueDataTransfer(request).get(); + return enqueueFileTransfer(request).get(); } -void DataTransfer::download(DataTransferRequest && request, Sink & sink) +void FileTransfer::download(FileTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the dataTransfer + that would cause the sink to execute on the fileTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. @@ -761,8 +761,8 @@ void DataTransfer::download(DataTransferRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDataTransfer(request, - {[_state](std::future fut) { + enqueueFileTransfer(request, + {[_state](std::future fut) { auto state(_state->lock()); state->quit = true; try { diff --git a/src/libstore/datatransfer.hh b/src/libstore/datatransfer.hh index 68d97ceb1..2347f363d 100644 --- a/src/libstore/datatransfer.hh +++ b/src/libstore/datatransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DataTransferSettings : Config +struct FileTransferSettings : Config { Setting enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,15 +31,15 @@ struct DataTransferSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DataTransferSettings dataTransferSettings; +extern FileTransferSettings fileTransferSettings; -struct DataTransferRequest +struct FileTransferRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = dataTransferSettings.tries; + size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -47,7 +47,7 @@ struct DataTransferRequest std::string mimeType; std::function dataCallback; - DataTransferRequest(const std::string & uri) + FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -56,7 +56,7 @@ struct DataTransferRequest } }; -struct DataTransferResult +struct FileTransferResult { bool cached = false; std::string etag; @@ -67,43 +67,43 @@ struct DataTransferResult class Store; -struct DataTransfer +struct FileTransfer { - virtual ~DataTransfer() { } + virtual ~FileTransfer() { } /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a DataTransferError + the download. The future may throw a FileTransferError exception. */ - virtual void enqueueDataTransfer(const DataTransferRequest & request, - Callback callback) = 0; + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback callback) = 0; - std::future enqueueDataTransfer(const DataTransferRequest & request); + std::future enqueueFileTransfer(const FileTransferRequest & request); /* Synchronously download a file. */ - DataTransferResult download(const DataTransferRequest & request); + FileTransferResult download(const FileTransferRequest & request); /* Synchronously upload a file. */ - DataTransferResult upload(const DataTransferRequest & request); + FileTransferResult upload(const FileTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DataTransferRequest && request, Sink & sink); + void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared DataTransfer object. Using this object is preferred +/* Return a shared FileTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref getDataTransfer(); +ref getFileTransfer(); -/* Return a new DataTransfer object. */ -ref makeDataTransfer(); +/* Return a new FileTransfer object. */ +ref makeFileTransfer(); -class DataTransferError : public Error +class FileTransferError : public Error { public: - DataTransfer::Error error; - DataTransferError(DataTransfer::Error error, const FormatOrString & fs) + FileTransfer::Error error; + FileTransferError(FileTransfer::Error error, const FormatOrString & fs) : Error(fs), error(error) { } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index c75846bc3..ce0c5af3b 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,14 +85,14 @@ protected: checkEnabled(); try { - DataTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - getDataTransfer()->download(request); + getFileTransfer()->download(request); return true; - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; @@ -103,19 +103,19 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DataTransferRequest(cacheUri + "/" + path); + auto req = FileTransferRequest(cacheUri + "/" + path); req.data = std::make_shared(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDataTransfer()->upload(req); - } catch (DataTransferError & e) { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - DataTransferRequest makeRequest(const std::string & path) + FileTransferRequest makeRequest(const std::string & path) { - DataTransferRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); return request; } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDataTransfer()->download(std::move(request), sink); - } catch (DataTransferError & e) { - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared(std::move(callback)); - getDataTransfer()->enqueueDataTransfer(request, - {[callbackPtr, this](std::future result) { + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future result) { try { (*callbackPtr)(result.get().data); - } catch (DataTransferError & e) { - if (e.error == DataTransfer::NotFound || e.error == DataTransfer::Forbidden) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr()); maybeDisable(); callbackPtr->rethrow(); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 478cb9f84..fccf010a4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -132,7 +132,7 @@ ref S3Helper::makeConfig(const string & region return res; } -S3Helper::DataTransferResult S3Helper::getObject( +S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +146,7 @@ S3Helper::DataTransferResult S3Helper::getObject( return Aws::New("STRINGSTREAM"); }); - DataTransferResult res; + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index d7d309243..2042bffcf 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DataTransferResult + struct FileTransferResult { std::shared_ptr data; unsigned int durationMs; }; - DataTransferResult getObject( + FileTransferResult getObject( const std::string & bucketName, const std::string & key); }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 001a3fb28..bb437cf1c 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -198,7 +198,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDataTransfer) + if (trusted || type == actFileTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 423ed95da..108b5dcb0 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -17,7 +17,7 @@ typedef enum { typedef enum { actUnknown = 0, actCopyPath = 100, - actDataTransfer = 101, + actFileTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 4c81e7d82..e136bb95a 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -180,9 +180,9 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DataTransferRequest req(actualUri); + FileTransferRequest req(actualUri); req.decompress = false; - getDataTransfer()->download(std::move(req), sink); + getFileTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ diff --git a/src/nix/main.cc b/src/nix/main.cc index 6bb04d8b6..140c963e0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -176,10 +176,10 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits::max(); - if (!dataTransferSettings.tries.overriden) - dataTransferSettings.tries = 0; - if (!dataTransferSettings.connectTimeout.overriden) - dataTransferSettings.connectTimeout = 1; + if (!fileTransferSettings.tries.overriden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overriden) + fileTransferSettings.connectTimeout = 1; } if (args.refresh) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index d108896ad..adc9b9a5d 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -190,8 +190,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDataTransfer && hasAncestor(*state, actCopyPath, parent)) - || (type == actDataTransfer && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +416,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDataTransfer, "%s MiB DL", "%.1f", MiB); + showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a4eda90b4..574f7d8c2 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -138,8 +138,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DataTransferRequest(storePathsUrl); - auto res = getDataTransfer()->download(req); + auto req = FileTransferRequest(storePathsUrl); + auto res = getFileTransfer()->download(req); auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); From f5095594e723d02c9bf22fe24ad838ebff4f9165 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Mon, 6 Apr 2020 23:57:28 +0200 Subject: [PATCH 114/350] datatransfer.{cc,hh} -> filetransfer.{cc,hh} --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/parser.y | 2 +- src/libstore/build.cc | 2 +- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/{datatransfer.cc => filetransfer.cc} | 2 +- src/libstore/{datatransfer.hh => filetransfer.hh} | 0 src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/nix-channel/nix-channel.cc | 2 +- src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- src/nix/main.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename src/libstore/{datatransfer.cc => filetransfer.cc} (99%) rename src/libstore/{datatransfer.hh => filetransfer.hh} (100%) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 26460601d..10c9c16bb 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,6 +1,6 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "util.hh" #include "eval.hh" #include "fetchers.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7a20c6fa6..b91a021b4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "json.hh" #include "function-trace.hh" diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8fbe59301..235ed5b1d 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -544,7 +544,7 @@ formal #include #include "eval.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "fetchers.hh" #include "store-api.hh" diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9baa2cf1a..0febb8dfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1432de9d7..486babf14 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" diff --git a/src/libstore/datatransfer.cc b/src/libstore/filetransfer.cc similarity index 99% rename from src/libstore/datatransfer.cc rename to src/libstore/filetransfer.cc index 5cc63a1d8..e9684b3d4 100644 --- a/src/libstore/datatransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,4 +1,4 @@ -#include "datatransfer.hh" +#include "filetransfer.hh" #include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/datatransfer.hh b/src/libstore/filetransfer.hh similarity index 100% rename from src/libstore/datatransfer.hh rename to src/libstore/filetransfer.hh diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ce0c5af3b..451a64785 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index fccf010a4..b24e7b7d6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "istringstream_nocopy.hh" #include diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 0f8024d7b..7d584f891 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "globals.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "../nix/legacy.hh" #include "fetchers.hh" diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index e136bb95a..748554b9c 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,6 +1,6 @@ #include "hash.hh" #include "shared.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 140c963e0..2c64c7476 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,7 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "finally.hh" #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 574f7d8c2..4fcc6a721 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "datatransfer.hh" +#include "filetransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" From 7867685dcdf1ba3c9290b777e0975ff0b775f667 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Wed, 8 Apr 2020 14:12:22 +0200 Subject: [PATCH 115/350] after flake rebase --- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/tarball.cc | 10 +++++----- src/nix-channel/nix-channel.cc | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 235ed5b1d..b33ab908b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -690,7 +690,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl try { res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9586f71ed..43c58485a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -2,7 +2,7 @@ #include "eval-inline.hh" #include "store-api.hh" #include "fetchers.hh" -#include "download.hh" +#include "filetransfer.hh" #include #include diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ef27eaa76..8675a5a66 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -1,4 +1,4 @@ -#include "download.hh" +#include "filetransfer.hh" #include "cache.hh" #include "fetchers.hh" #include "globals.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 4c4e5828e..695525b31 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,6 +1,6 @@ #include "fetchers.hh" #include "cache.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "store-api.hh" #include "archive.hh" @@ -36,13 +36,13 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - DownloadRequest request(url); + FileTransferRequest request(url); if (cached) request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); - DownloadResult res; + FileTransferResult res; try { - res = getDownloader()->download(request); - } catch (DownloadError & e) { + res = getFileTransfer()->download(request); + } catch (FileTransferError & e) { if (cached) { warn("%s; using cached version", e.msg()); return useCached(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 7d584f891..abd390414 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,7 +113,7 @@ static void update(const StringSet & channelNames) // Download the channel tarball. try { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (DataTransferError & e) { + } catch (FileTransferError & e) { filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } } From a693a9fa4b92df0d3283a9de8b1e968378a026d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 09:45:15 +0200 Subject: [PATCH 116/350] Attach pos to if expression errors --- src/libexpr/eval.cc | 2 +- src/libexpr/nixexpr.hh | 3 ++- src/libexpr/parser.y | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b91a021b4..f963a42ca 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1256,7 +1256,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); + (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8c96de37c..25798cac6 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -262,8 +262,9 @@ struct ExprWith : Expr struct ExprIf : Expr { + Pos pos; Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b33ab908b..1993fa6c1 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -335,7 +335,7 @@ expr_function ; expr_if - : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } | expr_op ; From a364b1551ae8724657369d68a09626c3cbef25d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 15:03:05 +0200 Subject: [PATCH 117/350] remote .travis.yml as we migrated to github actions --- .travis.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ee4ea1ac6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -matrix: - include: - - language: osx - script: ./tests/install-darwin.sh - - language: nix - script: nix-build release.nix -A build.x86_64-linux -notifications: - email: false From 30d4618cc944a41c2ca202babd0be0da4a1cd9d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 9 Apr 2020 15:04:51 +0200 Subject: [PATCH 118/350] README: add github actions badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 61054f8f2..9c0c87887 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) +[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix, the purely functional package manager ------------------------------------------ From 04bedda0b605dcecacb23b65d5a05c8f4aecfbdd Mon Sep 17 00:00:00 2001 From: Philipp Middendorf Date: Thu, 9 Apr 2020 17:05:29 +0200 Subject: [PATCH 119/350] gc.cc: Ignore hidden files in temproots --- src/libstore/gc.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0c3d89611..6bab1e37c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { + if (i.name[0] == '.') { + // Ignore hidden files. Some package managers (notably portage) create + // those to keep the directory alive. + continue; + } Path path = tempRootsDir + "/" + i.name; pid_t pid = std::stoi(i.name); From 16a4864759a80aa03b9c87b8aeaf7a4c31c03e39 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Sun, 17 Feb 2019 16:26:49 -0500 Subject: [PATCH 120/350] Delete temporary directory on successful build With --check and the --keep-failed (-K) flag, the temporary directory was being retained regardless of whether the build was successful and reproducible. This removes the temporary directory, as expected, on a reproducible check build. Added tests to verify that temporary build directories are not retained unnecessarily, particularly when using --check with --keep-failed. --- src/libstore/build.cc | 1 + tests/check.nix | 33 +++++++++++++++++++++++++++ tests/check.sh | 52 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0febb8dfb..760663ac9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1680,6 +1680,7 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { + deleteTmpDir(true); done(BuildResult::Built); return; } diff --git a/tests/check.nix b/tests/check.nix index 56c82e565..bca04fdaf 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -1,12 +1,45 @@ +{checkBuildId ? 0}: + with import ./config.nix; { nondeterministic = mkDerivation { + inherit checkBuildId; name = "nondeterministic"; buildCommand = '' mkdir $out date +%s.%N > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + deterministic = mkDerivation { + inherit checkBuildId; + name = "deterministic"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + failed = mkDerivation { + inherit checkBuildId; + name = "failed"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + false ''; }; diff --git a/tests/check.sh b/tests/check.sh index bc23a6634..b423dc0b5 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -1,14 +1,62 @@ source common.sh +checkBuildTempDirRemoved () +{ + buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1) + checkBuildIdFile=${buildDir}/checkBuildId + [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile +} + +# written to build temp directories to verify created by this instance +checkBuildId=$(date +%s%N) + clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check -nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +# check for dangling temporary build directories +# only retain if build fails and --keep-failed is specified, or... +# ...build is non-deterministic and --check and --keep-failed are both specified +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log +if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? + +# The above nix-build fails with status=1 on darwin (not sure why) +# ...but the primary purpose of the test case is to verify the temp directory is retained +if [ "$(uname -s)" != "Darwin" ]; then +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] +fi +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore From 3abf6d03c611417b309fdedf4323c08e6afbcd9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Apr 2020 17:27:35 +0200 Subject: [PATCH 121/350] Update release script --- maintainers/upload-release.pl | 86 ++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77534babb..cb584d427 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -1,5 +1,5 @@ #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 +#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1 use strict; use Data::Dumper; @@ -9,12 +9,16 @@ use File::Slurp; use File::Copy; use JSON::PP; use LWP::UserAgent; +use Net::Amazon::S3; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; -my $releasesDir = "/home/eelco/mnt/releases"; +my $releasesBucketName = "nix-releases"; +my $channelsBucketName = "nix-channels"; my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; +my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; + # FIXME: cut&paste from nixos-channel-scripts. sub fetch { my ($url, $type) = @_; @@ -42,13 +46,31 @@ my $version = $1; print STDERR "Nix revision is $nixRev, version is $version\n"; -File::Path::make_path($releasesDir); -if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; -} +my $releaseDir = "nix/$releaseName"; -my $releaseDir = "$releasesDir/nix/$releaseName"; -File::Path::make_path($releaseDir); +my $tmpDir = "$TMPDIR/nix-release/$releaseName"; +File::Path::make_path($tmpDir); + +# S3 setup. +my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given."; +my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given."; + +my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); + +my $releasesBucket = $s3->bucket($releasesBucketName) or die; + +my $s3_us = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + }); + +my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -57,40 +79,49 @@ sub downloadFile { my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; $dstName //= basename($srcFile); - my $dstFile = "$releaseDir/" . $dstName; + my $tmpFile = "$tmpDir/$dstName"; - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + if (!-e $tmpFile) { + print STDERR "downloading $srcFile to $tmpFile...\n"; + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0 or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; + print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } - write_file("$dstFile.sha256", $sha256_expected); + write_file("$tmpFile.sha256", $sha256_expected); - if (! -e "$dstFile.asc") { - system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; + if (! -e "$tmpFile.asc") { + system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n"; } - return ($dstFile, $sha256_expected); + return $sha256_expected; } downloadFile("tarball", "2"); # .tar.bz2 -my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz +my $tarballHash = downloadFile("tarball", "3"); # .tar.xz downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("installerScript", "1"); +for my $fn (glob "$tmpDir/*") { + my $name = basename($fn); + my $dstKey = "$releaseDir/" . $name; + unless (defined $releasesBucket->head_key($dstKey)) { + print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; + $releasesBucket->add_key_filename($dstKey, $fn) + or die $releasesBucket->err . ": " . $releasesBucket->errstr; + } +} + exit if $version =~ /pre/; # Update Nixpkgs in a very hacky way. @@ -125,18 +156,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; -# Extract the HTML manual. -File::Path::make_path("$releaseDir/manual"); - -system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die; - -if (! -e "$releaseDir/manual/index.html") { - symlink("manual.html", "$releaseDir/manual/index.html") or die; -} - # Update the "latest" symlink. -symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; -rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; +$channelsBucket->add_key( + "nix-latest/install", "", + { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" }) + or die $channelsBucket->err . ": " . $channelsBucket->errstr; # Tag the release in Git. chdir("/home/eelco/Dev/nix-pristine") or die; From 8132d0a12e1b1d30973ae2c543622a46c24ec075 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Sun, 17 Feb 2019 14:34:31 -0500 Subject: [PATCH 122/350] Fix nix-build --check -K in sandbox w/o root Temporarily add user-write permission to build directory so that it can be moved out of the sandbox to the store with a .check suffix. This is necessary because the build directory has already had its permissions set read-only, but write permission is required to update the directory's parent link to move it out of the sandbox. Updated the related --check "derivation may not be deterministic" messages to consistently use the real store paths. Added test for non-root sandbox nix-build --check -K to demonstrate issue and help prevent regressions. --- src/libstore/build.cc | 30 ++++++++++++++++++++++++++---- tests/linux-sandbox.sh | 7 +++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 760663ac9..b4207e1b8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3537,6 +3537,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv } +static void moveCheckToStore(const Path & src, const Path & dst) +{ + /* For the rename of directory to succeed, we must be running as root or + the directory must be made temporarily writable (to update the + directory's parent link ".."). */ + struct stat st; + if (lstat(src.c_str(), &st) == -1) { + throw SysError(format("getting attributes of path '%1%'") % src); + } + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % src % dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -3715,8 +3738,7 @@ void DerivationGoal::registerOutputs() if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + moveCheckToStore(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), @@ -3724,10 +3746,10 @@ void DerivationGoal::registerOutputs() path, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), path, dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), path); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); } /* Since we verified the build, it's now ultimately trusted. */ diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 52967d07d..16abd974c 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store + +# Test that sandboxed builds with --check and -K can move .check directory to store +nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link + +(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) +if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi +grep -q 'may not be deterministic' $TEST_ROOT/log From e8bd1bc732ff19f33e6915d14e15df1d7de612c0 Mon Sep 17 00:00:00 2001 From: Bruce Toll <4109762+tollb@users.noreply.github.com> Date: Fri, 10 Apr 2020 18:20:12 -0400 Subject: [PATCH 123/350] Add test case for temporary directories on darwin A test case for correct handling of temporary directory deletion that was added to check.sh as part of PR #2689 was initially disabled for Darwin because of a directory permission issue in PR #2688. Now that the issue in PR #2688 is fixed, this commit enables the test case for Darwin. --- tests/check.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/check.sh b/tests/check.sh index b423dc0b5..5f25d04cb 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -49,13 +49,8 @@ checkBuildTempDirRemoved $TEST_ROOT/log nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? - -# The above nix-build fails with status=1 on darwin (not sure why) -# ...but the primary purpose of the test case is to verify the temp directory is retained -if [ "$(uname -s)" != "Darwin" ]; then grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] -fi if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore From fc14213d2da7302b237dfe5ca51f6f4d9af34e5c Mon Sep 17 00:00:00 2001 From: DavHau <42246742+DavHau@users.noreply.github.com> Date: Wed, 25 Mar 2020 07:19:00 +0000 Subject: [PATCH 124/350] improve toFile error message when containing potential drv path --- src/libexpr/primops.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..6d7622900 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1021,7 +1021,9 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); + throw EvalError(format( + "in 'toFile': the file named '%1%' must not contain a reference " + "to a derivation but contains (%2%), at %3%") % name % path % pos); refs.insert(state.store->parseStorePath(path)); } From 4d9db420ffc9bd48da107a61c093b0d65d9d8db1 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 Apr 2020 09:57:22 +0200 Subject: [PATCH 125/350] never use /var/folders for TMPDIR on darwin This doesn't just cause problems for nix-store --serve but also results in certain build failures. Builds that use unix domain sockets in their tests often fail because the /var/folders prefix already consumes more than half of the maximum length of socket paths. struct sockaddr_un { sa_family_t sun_family; /* AF_UNIX */ char sun_path[108]; /* Pathname */ }; --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..72ba717e5 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -155,7 +155,7 @@ void initNix() sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ #if __APPLE__ - if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) unsetenv("TMPDIR"); #endif } From d2c371927e45c1d4e4f7e4ad331a3f68d67d5469 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 13 Apr 2020 21:09:47 +0200 Subject: [PATCH 126/350] SourceExprCommand: allocate the vSourceExpr via uncollectable memory Previously the memory would occasionally be collected during eval since the GC doesn't consider the member variable as alive / doesn't scan the region of memory where the pointer lives. By using the traceable_allocator allocator provided by Boehm GC we can ensure the memory isn't collected. It should be properly freed when SourceExprCommand goes out of scope. --- src/nix/command.hh | 2 +- src/nix/installables.cc | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index 23f5c9898..2c2303208 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -41,7 +41,7 @@ private: std::shared_ptr evalState; - Value * vSourceExpr = 0; + std::shared_ptr vSourceExpr; }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index f464d0aa1..902383bff 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,10 +8,13 @@ #include "store-api.hh" #include "shared.hh" +#include + #include namespace nix { + SourceExprCommand::SourceExprCommand() { mkFlag() @@ -24,11 +27,14 @@ SourceExprCommand::SourceExprCommand() Value * SourceExprCommand::getSourceExpr(EvalState & state) { - if (vSourceExpr) return vSourceExpr; + if (vSourceExpr) return vSourceExpr.get(); auto sToplevel = state.symbols.create("_toplevel"); - vSourceExpr = state.allocValue(); + // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't + // consider the member variable "alive" during execution causing it to be + // GC'ed in the middle of evaluation. + vSourceExpr = std::allocate_shared(traceable_allocator()); if (file != "") state.evalFile(lookupFileArg(state, file), *vSourceExpr); @@ -69,7 +75,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) vSourceExpr->attrs->sort(); } - return vSourceExpr; + return vSourceExpr.get(); } ref SourceExprCommand::getEvalState() From 895516cadf0739dd8359a38d503e0aaf5068dc4b Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 30 Mar 2020 15:31:14 +0200 Subject: [PATCH 127/350] add NIX_USER_CONF_FILES Motivation: maintain project-level configuration files. Document the whole situation a bit better so that it corresponds to the implementation, and add NIX_USER_CONF_FILES that allows overriding which user files Nix will load during startup. --- doc/manual/command-ref/conf-file.xml | 36 ++++++++++++++----------- doc/manual/command-ref/env-common.xml | 13 ++++++--- src/libmain/shared.cc | 5 +++- src/libstore/globals.cc | 25 ++++++++++++++--- src/libstore/globals.hh | 8 +++++- src/libutil/util.hh | 1 + tests/common.sh.in | 3 +++ tests/config.sh | 18 +++++++++++++ tests/config/nix-with-substituters.conf | 2 ++ tests/local.mk | 1 + 10 files changed, 87 insertions(+), 25 deletions(-) create mode 100644 tests/config.sh create mode 100644 tests/config/nix-with-substituters.conf diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 48dce7c95..1820598e5 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -19,26 +19,30 @@ Description -Nix reads settings from two configuration files: +By default Nix reads settings from the following places: - +The system-wide configuration file +sysconfdir/nix/nix.conf +(i.e. /etc/nix/nix.conf on most systems), or +$NIX_CONF_DIR/nix.conf if +NIX_CONF_DIR is set. Values loaded in this file are not forwarded to the Nix daemon. The +client assumes that the daemon has already loaded them. + - - The system-wide configuration file - sysconfdir/nix/nix.conf - (i.e. /etc/nix/nix.conf on most systems), or - $NIX_CONF_DIR/nix.conf if - NIX_CONF_DIR is set. - +User-specific configuration files: - - The user configuration file - $XDG_CONFIG_HOME/nix/nix.conf, or - ~/.config/nix/nix.conf if - XDG_CONFIG_HOME is not set. - + + If NIX_USER_CONF_FILES is set, then each path separated by + : will be loaded in reverse order. + - + + Otherwise it will look for nix/nix.conf files in + XDG_CONFIG_DIRS and XDG_CONFIG_HOME. + + The default location is $HOME/.config/nix.conf if + those environment variables are unset. + The configuration files consist of name = diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 696d68c34..0217de7b2 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -33,7 +33,7 @@ will cause Nix to look for paths relative to /home/eelco/Dev and - /etc/nixos, in that order. It is also + /etc/nixos, in this order. It is also possible to match paths against a prefix. For example, the value @@ -59,7 +59,7 @@ nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz A following shorthand can be used to refer to the official channels: - + nixpkgs=channel:nixos-15.09 @@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix NIX_CONF_DIR - Overrides the location of the Nix configuration + Overrides the location of the system Nix configuration directory (default prefix/etc/nix). +NIX_USER_CONF_FILES + + Overrides the location of the user Nix configuration files + to load from (defaults to the XDG spec locations). The variable is treated + as a list separated by the : token. + + TMPDIR diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..3c141a5fa 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -260,7 +260,10 @@ void printVersion(const string & programName) cfg.push_back("signed-caches"); #endif std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "User configuration files: " << + concatStringsSep(":", settings.nixUserConfFiles) + << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e97f3c22..d6ea0318e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -31,6 +31,7 @@ Settings::Settings() , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixUserConfFiles(getUserConfigFiles()) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) @@ -77,13 +78,29 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); - auto dirs = getConfigDirs(); - // Iterate over them in reverse so that the ones appearing first in the path take priority - for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { - globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + auto files = settings.nixUserConfFiles; + for (auto file = files.rbegin(); file != files.rend(); file++) { + globalConfig.applyConfigFile(*file); } } +std::vector getUserConfigFiles() +{ + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); + if (nixConfFiles.has_value()) { + return tokenizeString>(nixConfFiles.value(), ":"); + } + + // Use the paths specified by the XDG spec + std::vector files; + auto dirs = getConfigDirs(); + for (auto & dir : dirs) { + files.insert(files.end(), dir + "/nix/nix.conf"); + } + return files; +} + unsigned int Settings::getDefaultCores() { return std::max(1U, std::thread::hardware_concurrency()); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 40f350f0b..da95fd3ae 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -53,9 +53,12 @@ public: /* The directory where state is stored. */ Path nixStateDir; - /* The directory where configuration files are stored. */ + /* The directory where system configuration files are stored. */ Path nixConfDir; + /* A list of user configuration files to load. */ + std::vector nixUserConfFiles; + /* The directory where internal helper programs are stored. */ Path nixLibexecDir; @@ -378,6 +381,9 @@ void initPlugins(); void loadConfFile(); +// Used by the Settings constructor +std::vector getUserConfigFiles(); + extern const string nixVersion; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1f85c7c46..815b1f288 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -16,6 +16,7 @@ #include #include #include +#include #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 diff --git a/tests/common.sh.in b/tests/common.sh.in index 15d7b1ef9..dd7e61822 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc +unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 @@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME mkdir -p $TEST_HOME diff --git a/tests/config.sh b/tests/config.sh new file mode 100644 index 000000000..8fa349f11 --- /dev/null +++ b/tests/config.sh @@ -0,0 +1,18 @@ +source common.sh + +# Test that files are loaded from XDG by default +export XDG_CONFIG_HOME=/tmp/home +export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] + +# Test that setting NIX_USER_CONF_FILES overrides all the default user config files +export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] + +# Test that it's possible to load the config from a custom location +here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf +var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +[[ $var == https://example.com ]] diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf new file mode 100644 index 000000000..90f359a6f --- /dev/null +++ b/tests/config/nix-with-substituters.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +substituters = https://example.com diff --git a/tests/local.mk b/tests/local.mk index 01fac4fcd..c5602773d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,6 @@ nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ + config.sh \ gc.sh \ gc-concurrent.sh \ gc-auto.sh \ From 057e5b6b2e8bf7977c1f57d6f80cc17aa40096d9 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 15 Apr 2020 10:09:43 -0600 Subject: [PATCH 128/350] move implementation to cc --- src/libutil/error.cc | 5 +++++ src/libutil/error.hh | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 40354cc87..a5571d4ec 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -8,6 +8,11 @@ namespace nix std::optional ErrorInfo::programName = std::nullopt; +std::ostream& operator<<(std::ostream &os, const hintformat &hf) +{ + return os << hf.str(); +} + string showErrPos(const ErrPos &errPos) { if (errPos.column > 0) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 8286eaab1..5658e6335 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -92,10 +92,7 @@ private: format fmt; }; -std::ostream& operator<<(std::ostream &os, const hintformat &hf) -{ - return os << hf.str(); -} +std::ostream& operator<<(std::ostream &os, const hintformat &hf); template inline hintformat hintfmt(const std::string & fs, const Args & ... args) From b865b5b40c13cfff32e1a0ab685baff1bef5ae5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 16 Apr 2020 12:32:07 +0200 Subject: [PATCH 129/350] pass Pos to forceValue to improve infinite recursion error --- src/libexpr/eval-inline.hh | 4 +-- src/libexpr/eval.cc | 6 ++-- src/libexpr/primops.cc | 64 +++++++++++++++++++------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c27116e3b..942cda1ea 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -57,7 +57,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tAttrs) throwTypeError("value is %1% while a set was expected, at %2%", v, pos); } @@ -73,7 +73,7 @@ inline void EvalState::forceList(Value & v) inline void EvalState::forceList(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (!v.isList()) throwTypeError("value is %1% while a list was expected, at %2%", v, pos); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f963a42ca..3b4d9bfdc 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1502,7 +1502,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tBool) throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); return v.boolean; @@ -1517,7 +1517,7 @@ bool EvalState::isFunctor(Value & fun) void EvalState::forceFunction(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) throwTypeError("value is %1% while a function was expected, at %2%", v, pos); } @@ -1594,7 +1594,7 @@ std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { - forceValue(v); + forceValue(v, pos); string s; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc6c8296b..a00b93676 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -241,7 +241,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); string t; switch (args[0]->type) { case tInt: t = "int"; break; @@ -269,7 +269,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tNull); } @@ -277,7 +277,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); bool res; switch (args[0]->type) { case tLambda: @@ -296,21 +296,21 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tInt); } /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tFloat); } /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tString); } @@ -318,14 +318,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tBool); } /* Determine whether the argument is a path. */ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tPath); } @@ -382,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) throw EvalError(format("attribute 'operator' required, at %1%") % pos); - state.forceValue(*op->value); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of `workSet', adding the result to `workSet', continuing until @@ -401,7 +401,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) throw EvalError(format("attribute 'key' required, at %1%") % pos); - state.forceValue(*key->value); + state.forceValue(*key->value, pos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); @@ -413,7 +413,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Add the values returned by the operator to the work set. */ for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n]); + state.forceValue(*call.listElems()[n], pos); workSet.push_back(call.listElems()[n]); } } @@ -445,7 +445,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { try { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { PathSet context; @@ -461,7 +461,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val { state.mkAttrs(v, 2); try { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); v.attrs->push_back(Attr(state.sValue, args[0])); mkBool(*state.allocAttr(v, state.symbols.create("success")), true); } catch (AssertionError & e) { @@ -483,8 +483,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -494,7 +494,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValueDeep(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -503,12 +503,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val return the second expression. Useful for debugging. */ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type == tString) printError(format("trace: %1%") % args[0]->string.s); else printError(format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -599,7 +599,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * try { if (ignoreNulls) { - state.forceValue(*i->value); + state.forceValue(*i->value, pos); if (i->value->type == tNull) continue; } @@ -1092,7 +1092,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args if (!context.empty()) throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); @@ -1118,7 +1118,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { - state.forceValue(*attr.value); + state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") recursive = state.forceBool(*attr.value, *attr.pos); @@ -1189,7 +1189,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); + state.forceValue(*i->value, pos); v = *i->value; } @@ -1219,7 +1219,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tAttrs); } @@ -1345,7 +1345,7 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va */ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); @@ -1391,7 +1391,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a list. */ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->isList()); } @@ -1401,7 +1401,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.listElems()[n]); + state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -1524,9 +1524,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } - state.forceValue(v); + state.forceValue(v, pos); } else { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } } @@ -1591,7 +1591,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto len = args[1]->listSize(); state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n]); + state.forceValue(*args[1]->listElems()[n], pos); v.listElems()[n] = args[1]->listElems()[n]; } @@ -1626,7 +1626,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < len; ++n) { auto vElem = args[1]->listElems()[n]; - state.forceValue(*vElem); + state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); if (state.forceBool(res, pos)) @@ -1757,8 +1757,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); CompareValues comp; mkBool(v, comp(args[0], args[1])); } From 1290411c2d0c62dd1761485f55292dc944eae55d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Apr 2020 14:00:12 +0200 Subject: [PATCH 130/350] fetchMercurial: Use inputFromAttrs() --- src/libexpr/primops/fetchMercurial.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index f18351646..0a1ba49d5 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -54,15 +54,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar if (evalSettings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - auto parsedUrl = parseURL( - url.find("://") != std::string::npos - ? "hg+" + url - : "hg+file://" + url); - if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); - if (ref) parsedUrl.query.insert_or_assign("ref", *ref); - // FIXME: use name - auto input = fetchers::inputFromURL(parsedUrl); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "hg"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + auto input = fetchers::inputFromAttrs(attrs); + // FIXME: use name auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); From b3e5eea4a91400fb2a12aba4b07a94d03ba54605 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 16:28:07 +0200 Subject: [PATCH 131/350] Add function to allocate a Value in traceable memory --- src/libexpr/eval.cc | 8 ++++++++ src/libexpr/value.hh | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f963a42ca..2f5b1db47 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -22,6 +22,8 @@ #if HAVE_BOEHMGC +#define GC_INCLUDE_NEW + #include #include @@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size) } +RootValue allocRootValue(Value * v) +{ + return std::allocate_shared(traceable_allocator(), v); +} + + static void printValue(std::ostream & str, std::set & active, const Value & v) { checkInterrupt(); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 689373873..9cf2bf06a 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -261,4 +261,9 @@ typedef std::map ValueMap; #endif +/* A value allocated in traceable memory. */ +typedef std::shared_ptr RootValue; + +RootValue allocRootValue(Value * v); + } From 10e17eaa5802a3c368eee8408821828072e76ba7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 17:24:28 +0200 Subject: [PATCH 132/350] ValueMap, VectorVector: Use traceable_allocator We want to *trace* the 'Value *' arrays, not garbage-collect them! Otherwise the vectors/maps can end up pointing to nowhere. Fixes #3377. Closes #3384. --- src/libexpr/value.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 9cf2bf06a..71025824e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -253,8 +253,8 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC -typedef std::vector > ValueVector; -typedef std::map, gc_allocator > > ValueMap; +typedef std::vector > ValueVector; +typedef std::map, traceable_allocator > > ValueMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; From 9f46f54de4e55267df492456fc0393f74616366b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 17:28:32 +0200 Subject: [PATCH 133/350] JSONSax: Use a RootValue More #3377. --- src/libexpr/json-to-value.cc | 63 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 1fdce1983..76e1a26bf 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -4,7 +4,6 @@ #include using json = nlohmann::json; -using std::unique_ptr; namespace nix { @@ -13,69 +12,69 @@ namespace nix { class JSONSax : nlohmann::json_sax { class JSONState { protected: - unique_ptr parent; - Value * v; + std::unique_ptr parent; + RootValue v; public: - virtual unique_ptr resolve(EvalState &) + virtual std::unique_ptr resolve(EvalState &) { throw std::logic_error("tried to close toplevel json parser state"); - }; - explicit JSONState(unique_ptr&& p) : parent(std::move(p)), v(nullptr) {}; - explicit JSONState(Value* v) : v(v) {}; - JSONState(JSONState& p) = delete; - Value& value(EvalState & state) + } + explicit JSONState(std::unique_ptr && p) : parent(std::move(p)) {} + explicit JSONState(Value * v) : v(allocRootValue(v)) {} + JSONState(JSONState & p) = delete; + Value & value(EvalState & state) { - if (v == nullptr) - v = state.allocValue(); - return *v; - }; - virtual ~JSONState() {}; - virtual void add() {}; + if (!v) + v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} }; class JSONObjectState : public JSONState { using JSONState::JSONState; - ValueMap attrs = ValueMap(); - virtual unique_ptr resolve(EvalState & state) override + ValueMap attrs; + std::unique_ptr resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkAttrs(v, attrs.size()); for (auto & i : attrs) v.attrs->push_back(Attr(i.first, i.second)); return std::move(parent); } - virtual void add() override { v = nullptr; }; + void add() override { v = nullptr; } public: - void key(string_t& name, EvalState & state) + void key(string_t & name, EvalState & state) { - attrs[state.symbols.create(name)] = &value(state); + attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; class JSONListState : public JSONState { - ValueVector values = ValueVector(); - virtual unique_ptr resolve(EvalState & state) override + ValueVector values; + std::unique_ptr resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkList(v, values.size()); for (size_t n = 0; n < values.size(); ++n) { v.listElems()[n] = values[n]; } return std::move(parent); } - virtual void add() override { - values.push_back(v); + void add() override { + values.push_back(*v); v = nullptr; - }; + } public: - JSONListState(unique_ptr&& p, std::size_t reserve) : JSONState(std::move(p)) + JSONListState(std::unique_ptr && p, std::size_t reserve) : JSONState(std::move(p)) { values.reserve(reserve); } }; EvalState & state; - unique_ptr rs; + std::unique_ptr rs; template inline bool handle_value(T f, Args... args) { @@ -107,12 +106,12 @@ public: return handle_value(mkInt, val); } - bool number_float(number_float_t val, const string_t& s) + bool number_float(number_float_t val, const string_t & s) { return handle_value(mkFloat, val); } - bool string(string_t& val) + bool string(string_t & val) { return handle_value(mkString, val.c_str()); } @@ -123,7 +122,7 @@ public: return true; } - bool key(string_t& name) + bool key(string_t & name) { dynamic_cast(rs.get())->key(name, state); return true; From 96262e744e4eef67d808a532b1704fceb62bca5b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 16 Apr 2020 09:55:38 -0600 Subject: [PATCH 134/350] switch to structs, which don't need public: --- src/libutil/error.hh | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 5658e6335..417a6ad95 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -17,9 +17,8 @@ typedef enum { elError } ErrLevel; -class ErrPos +struct ErrPos { -public: int lineNumber; int column; string nixFile; @@ -40,9 +39,8 @@ public: } }; -class NixCode +struct NixCode { -public: ErrPos errPos; std::optional prevLineOfCode; string errLineOfCode; @@ -54,9 +52,8 @@ public: // are always in yellow. template -class yellowify +struct yellowify { -public: yellowify(T &s) : value(s) {} T &value; }; @@ -104,9 +101,8 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) // ------------------------------------------------- // ErrorInfo. -class ErrorInfo +struct ErrorInfo { -public: ErrLevel level; string name; string description; @@ -114,8 +110,6 @@ public: std::optional nixCode; static std::optional programName; - -private: }; // -------------------------------------------------------- From fcd048a526bd239fa615457e77d61d69d679bf03 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 16:54:34 +0200 Subject: [PATCH 135/350] Use RootValue --- src/libexpr/primops.cc | 10 +++++----- src/nix/command.hh | 2 +- src/nix/installables.cc | 21 ++++++++------------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc6c8296b..a3f2b92ce 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -121,16 +121,16 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } w.attrs->sort(); - static Value * fun = nullptr; + static RootValue fun; if (!fun) { - fun = state.allocValue(); + fun = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), *fun); + , "/"), **fun); } - state.forceFunction(*fun, pos); - mkApp(v, *fun, w); + state.forceFunction(**fun, pos); + mkApp(v, **fun, w); state.forceAttrs(v, pos); } else { state.forceAttrs(*args[0]); diff --git a/src/nix/command.hh b/src/nix/command.hh index 2c2303208..bf43d950f 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -41,7 +41,7 @@ private: std::shared_ptr evalState; - std::shared_ptr vSourceExpr; + RootValue vSourceExpr; }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 902383bff..1d70ad3d5 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,8 +8,6 @@ #include "store-api.hh" #include "shared.hh" -#include - #include namespace nix { @@ -27,17 +25,14 @@ SourceExprCommand::SourceExprCommand() Value * SourceExprCommand::getSourceExpr(EvalState & state) { - if (vSourceExpr) return vSourceExpr.get(); + if (vSourceExpr) return *vSourceExpr; auto sToplevel = state.symbols.create("_toplevel"); - // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't - // consider the member variable "alive" during execution causing it to be - // GC'ed in the middle of evaluation. - vSourceExpr = std::allocate_shared(traceable_allocator()); + vSourceExpr = allocRootValue(state.allocValue()); if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); + state.evalFile(lookupFileArg(state, file), **vSourceExpr); else { @@ -45,9 +40,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, 1024); + state.mkAttrs(**vSourceExpr, 1024); - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); std::unordered_set seen; @@ -58,7 +53,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), + mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); }; @@ -72,10 +67,10 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) } else addEntry(i.first); - vSourceExpr->attrs->sort(); + (*vSourceExpr)->attrs->sort(); } - return vSourceExpr.get(); + return *vSourceExpr; } ref SourceExprCommand::getEvalState() From 67a5941472ab56a798556ee2d5afc62fc38a799a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 13:12:58 +0200 Subject: [PATCH 136/350] Logger: Add method for writing to stdout Usually this just writes to stdout, but for ProgressBar, we need to clear the current line, write the line to stdout, and then redraw the progress bar. (cherry picked from commit 696c026006a6ac46adc990ed5cb0f31535bac076) --- src/libutil/logging.cc | 6 ++++++ src/libutil/logging.hh | 10 ++++++++++ src/nix/progress-bar.cc | 13 +++++++++++++ 3 files changed, 29 insertions(+) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index bb437cf1c..3cc4ef8f1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,6 +3,7 @@ #include #include +#include namespace nix { @@ -24,6 +25,11 @@ void Logger::warn(const std::string & msg) log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); } +void Logger::writeToStdout(std::string_view s) +{ + std::cout << s << "\n"; +} + class SimpleLogger : public Logger { public: diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 108b5dcb0..18c24d508 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -78,6 +78,16 @@ public: virtual void stopActivity(ActivityId act) { }; virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + + virtual void writeToStdout(std::string_view s); + + template + inline void stdout(const std::string & fs, const Args & ... args) + { + boost::format f(fs); + formatHelper(f, args...); + writeToStdout(f.str()); + } }; ActivityId getCurActivity(); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index adc9b9a5d..8e7ba95a3 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -7,6 +7,7 @@ #include #include #include +#include namespace nix { @@ -442,6 +443,18 @@ public: return res; } + + void writeToStdout(std::string_view s) override + { + auto state(state_.lock()); + if (state->active) { + std::cerr << "\r\e[K"; + Logger::writeToStdout(s); + draw(*state); + } else { + Logger::writeToStdout(s); + } + } }; void startProgressBar(bool printBuildLogs) From efaffaa9d1de38efecb718aa7a99ba1f2e342ade Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Apr 2020 13:46:37 +0200 Subject: [PATCH 137/350] Use Logger::stdout() (cherry picked from commit 8f41847394524fcac40d3b5620139ca7e94a18e3) --- src/nix/add-to-store.cc | 2 +- src/nix/eval.cc | 5 ++--- src/nix/hash.cc | 5 ++--- src/nix/ls.cc | 10 ++++------ src/nix/show-config.cc | 2 +- src/nix/why-depends.cc | 2 +- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 139db3657..ed02227db 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!dryRun) store->addToStore(info, sink.s); - std::cout << fmt("%s\n", store->printStorePath(info.path)); + logger->stdout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 6398fc58e..86a1e8b68 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -55,16 +55,15 @@ struct CmdEval : MixJSON, InstallableCommand auto v = installable->toValue(*state).first; PathSet context; - stopProgressBar(); - if (raw) { + stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); } else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - std::cout << *v << "\n"; + logger->stdout("%s", *v); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0cc523f50..01628cf6c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -60,8 +60,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - h.to_string(base, base == SRI); + logger->stdout(h.to_string(base, base == SRI)); } } }; @@ -95,7 +94,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3ef1f2750..8590199d7 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); + auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; + line += " -> " + accessor->readLink(curPath); + logger->stdout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - std::cout << relPath << "\n"; + logger->stdout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 87544f937..6104b10bc 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -23,7 +23,7 @@ struct CmdShowConfig : Command, MixJSON std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) - std::cout << s.first + " = " + s.second.value + "\n"; + logger->stdout("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index d3b7a674a..f9acc7f13 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -149,7 +149,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", + logger->stdout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", From 12814806efca65a73aebf8d3109eb2fbf880e2b8 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 16 Apr 2020 10:48:15 -0600 Subject: [PATCH 138/350] iomanip no longer needed --- src/libutil/error.hh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 417a6ad95..f402b692e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -5,8 +5,6 @@ #include #include #include -#include - #include "types.hh" namespace nix From 3d5b1032a193419cfa7406f0e42a7621994d70af Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 17 Apr 2020 15:07:44 -0600 Subject: [PATCH 139/350] logError, logWarning; Logger functions; switch to Verbosity enum --- src/error-demo/error-demo.cc | 67 ++++++++++++++++++------------------ src/libstore/daemon.cc | 4 +++ src/libutil/error.cc | 50 +++++++++++++++++++-------- src/libutil/error.hh | 19 +++++----- src/libutil/logging.cc | 18 ++++++++++ src/libutil/logging.hh | 30 ++++++++++------ src/nix/progress-bar.cc | 6 ++++ 7 files changed, 128 insertions(+), 66 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index a9ff6057c..b7dd4cc22 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -1,4 +1,4 @@ -#include "error.hh" +#include "logging.hh" #include "nixexpr.hh" #include @@ -8,24 +8,25 @@ int main() { using namespace nix; + std::unique_ptr logger(makeDefaultLogger()); + + verbosity = lvlError; + // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); // Error in a program; no hint and no nix code. - printErrorInfo( - ErrorInfo { .level = elError, - .name = "name", + logError( + ErrorInfo { .name = "name", .description = "error description", }); // Warning with name, description, and hint. // The hintfmt function makes all the substituted text yellow. - printErrorInfo( - ErrorInfo { .level = elWarning, - .name = "name", + logWarning( + ErrorInfo { .name = "name", .description = "error description", - .hint = std::optional( - hintfmt("there was a %1%", "warning")), + .hint = hintfmt("there was a %1%", "warning"), }); @@ -34,32 +35,32 @@ int main() SymbolTable testTable; auto problem_file = testTable.create("myfile.nix"); - printErrorInfo( - ErrorInfo{ - .level = elWarning, - .name = "warning name", - .description = "warning description", - .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::nullopt, - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::nullopt - }}); + logWarning( + ErrorInfo { .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); // Error with previous and next lines of code. - printErrorInfo( - ErrorInfo{ - .level = elError, - .name = "error name", - .description = "error description", - .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::optional("previous line of code"), - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::optional("next line of code"), - }}); + logError( + ErrorInfo { .name = "error name", + .description = "error description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::optional("previous line of code"), + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::optional("next line of code"), + }}); return 0; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8e9f9d71b..c8daef57c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -73,6 +73,10 @@ struct TunnelLogger : public Logger enqueueMsg(*buf.s); } + void logEI(const ErrorInfo & ei) override + { + } + /* startWork() means that we're starting an operation for which we want to send out stderr to the client. */ void startWork() diff --git a/src/libutil/error.cc b/src/libutil/error.cc index a5571d4ec..779f519c0 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -2,6 +2,7 @@ #include #include +#include "serialise.hh" namespace nix { @@ -66,25 +67,45 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) } } -void printErrorInfo(const ErrorInfo &einfo) +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) { int errwidth = 80; string prefix = " "; string levelString; switch (einfo.level) { - case ErrLevel::elError: { + case Verbosity::lvlError: { levelString = ANSI_RED; levelString += "error:"; levelString += ANSI_NORMAL; break; } - case ErrLevel::elWarning: { + case Verbosity::lvlWarn: { levelString = ANSI_YELLOW; levelString += "warning:"; levelString += ANSI_NORMAL; break; } + case Verbosity::lvlInfo: { + levelString = ANSI_YELLOW; + levelString += "info:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlTalkative: + case Verbosity::lvlChatty: + case Verbosity::lvlVomit: { + levelString = ANSI_GREEN; + levelString += "info:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlDebug: { + levelString = ANSI_YELLOW; + levelString += "debug:"; + levelString += ANSI_NORMAL; + break; + } default: { levelString = fmt("invalid error level: %1%", einfo.level); break; @@ -99,7 +120,7 @@ void printErrorInfo(const ErrorInfo &einfo) dashes.append("-"); // divider. - std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL, + out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL, prefix, levelString, "---", @@ -115,32 +136,33 @@ void printErrorInfo(const ErrorInfo &einfo) ? string(" ") + showErrPos(einfo.nixCode->errPos) : ""; - std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL, + out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL, prefix, einfo.nixCode->errPos.nixFile, eline) << std::endl; - std::cout << prefix << std::endl; + out << prefix << std::endl; } else { - std::cout << fmt("%1%from command line argument", prefix) << std::endl; - std::cout << prefix << std::endl; + out << fmt("%1%from command line argument", prefix) << std::endl; + out << prefix << std::endl; } } // description - std::cout << prefix << einfo.description << std::endl; - std::cout << prefix << std::endl; + out << prefix << einfo.description << std::endl; + out << prefix << std::endl; // lines of code. if (einfo.nixCode->errLineOfCode != "") { printCodeLines(prefix, *einfo.nixCode); - std::cout << prefix << std::endl; + out << prefix << std::endl; } // hint if (einfo.hint.has_value()) { - std::cout << prefix << *einfo.hint << std::endl; - std::cout << prefix << std::endl; + out << prefix << *einfo.hint << std::endl; + out << prefix << std::endl; } -} + return out; +} } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f402b692e..0419a1b52 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -11,9 +11,14 @@ namespace nix { typedef enum { - elWarning, - elError -} ErrLevel; + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; struct ErrPos { @@ -101,7 +106,7 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) // ErrorInfo. struct ErrorInfo { - ErrLevel level; + Verbosity level; string name; string description; std::optional hint; @@ -110,11 +115,7 @@ struct ErrorInfo static std::optional programName; }; -// -------------------------------------------------------- -// error printing - -// just to cout for now. -void printErrorInfo(const ErrorInfo &einfo); +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index fa5c84a27..98f500a65 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,6 +3,7 @@ #include #include +#include namespace nix { @@ -57,6 +58,14 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } + void logEI(const ErrorInfo & ei) override + { + std::stringstream oss; + oss << ei; + + log(ei.level, oss.str()); + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override @@ -135,6 +144,15 @@ struct JSONLogger : Logger write(json); } + void logEI(const ErrorInfo & ei) override + { + // nlohmann::json json; + // json["action"] = "msg"; + // json["level"] = lvl; + // json["msg"] = fs.s; + // write(json); + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index beb5e6b64..ba6f39ac9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,19 +1,10 @@ #pragma once #include "types.hh" +#include "error.hh" namespace nix { -typedef enum { - lvlError = 0, - lvlWarn, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - typedef enum { actUnknown = 0, actCopyPath = 100, @@ -70,6 +61,13 @@ public: log(lvlInfo, fs); } + virtual void logEI(const ErrorInfo &ei) = 0; + + void logEI(Verbosity lvl, ErrorInfo ei) { + ei.level = lvl; + logEI(ei); + } + virtual void warn(const std::string & msg); virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -157,6 +155,18 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) +#define logErrorInfo(level, errorInfo...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->logEI(level, errorInfo); \ + } \ + } while (0) + +#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) +#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) + + + template inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 26631416c..41f3d4cd7 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -124,6 +124,12 @@ public: log(*state, lvl, fs.s); } + void logEI(const ErrorInfo &ei) override + { + auto state(state_.lock()); + // log(*state, lvl, ei.as_str()); + } + void log(State & state, Verbosity lvl, const std::string & s) { if (state.active) { From 4697552948b4d9b958b0e8f8ff1b68b5e9608e4e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 17 Apr 2020 15:50:46 -0600 Subject: [PATCH 140/350] demoing other error levels than warn/error; rename line and file fields in errPos --- src/error-demo/error-demo.cc | 37 ++++++++++++++++++++++++-- src/libutil/error.cc | 50 +++++++++++++++++++++--------------- src/libutil/error.hh | 8 +++--- src/libutil/logging.cc | 39 ++++++++++++++++++++++++---- 4 files changed, 103 insertions(+), 31 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index b7dd4cc22..14027278d 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -8,13 +8,46 @@ int main() { using namespace nix; - std::unique_ptr logger(makeDefaultLogger()); + makeDefaultLogger(); - verbosity = lvlError; + verbosity = lvlVomit; // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); + // For completeness sake, info through vomit levels. + // But this is maybe a heavy format for those. + logger->logEI( + ErrorInfo { .level = lvlInfo, + .name = "Info name", + .description = "Info description", + }); + + logger->logEI( + ErrorInfo { .level = lvlTalkative, + .name = "Talkative name", + .description = "Talkative description", + }); + + logger->logEI( + ErrorInfo { .level = lvlChatty, + .name = "Chatty name", + .description = "Chatty description", + }); + + logger->logEI( + ErrorInfo { .level = lvlDebug, + .name = "Debug name", + .description = "Debug description", + }); + + logger->logEI( + ErrorInfo { .level = lvlVomit, + .name = "Vomit name", + .description = "Vomit description", + }); + + // Error in a program; no hint and no nix code. logError( ErrorInfo { .name = "name", diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 779f519c0..0e07c0d98 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -17,9 +17,9 @@ std::ostream& operator<<(std::ostream &os, const hintformat &hf) string showErrPos(const ErrPos &errPos) { if (errPos.column > 0) { - return fmt("(%1%:%2%)", errPos.lineNumber, errPos.column); + return fmt("(%1%:%2%)", errPos.line, errPos.column); } else { - return fmt("(%1%)", errPos.lineNumber); + return fmt("(%1%)", errPos.line); }; } @@ -29,7 +29,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) if (nixCode.prevLineOfCode.has_value()) { std::cout << fmt("%1% %|2$5d|| %3%", prefix, - (nixCode.errPos.lineNumber - 1), + (nixCode.errPos.line - 1), *nixCode.prevLineOfCode) << std::endl; } @@ -37,7 +37,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) // line of code containing the error.%2$+5d% std::cout << fmt("%1% %|2$5d|| %3%", prefix, - (nixCode.errPos.lineNumber), + (nixCode.errPos.line), nixCode.errLineOfCode) << std::endl; @@ -61,7 +61,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) if (nixCode.nextLineOfCode.has_value()) { std::cout << fmt("%1% %|2$5d|| %3%", prefix, - (nixCode.errPos.lineNumber + 1), + (nixCode.errPos.line + 1), *nixCode.nextLineOfCode) << std::endl; } @@ -87,16 +87,26 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) break; } case Verbosity::lvlInfo: { - levelString = ANSI_YELLOW; + levelString = ANSI_GREEN; levelString += "info:"; levelString += ANSI_NORMAL; break; } - case Verbosity::lvlTalkative: - case Verbosity::lvlChatty: + case Verbosity::lvlTalkative: { + levelString = ANSI_GREEN; + levelString += "talk:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlChatty: { + levelString = ANSI_GREEN; + levelString += "chat:"; + levelString += ANSI_NORMAL; + break; + } case Verbosity::lvlVomit: { levelString = ANSI_GREEN; - levelString += "info:"; + levelString += "vomit:"; levelString += ANSI_NORMAL; break; } @@ -121,25 +131,25 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) // divider. out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL, - prefix, - levelString, - "---", - einfo.name, - dashes, - einfo.programName.value_or("")) - << std::endl; + prefix, + levelString, + "---", + einfo.name, + dashes, + einfo.programName.value_or("")) + << std::endl; // filename. if (einfo.nixCode.has_value()) { - if (einfo.nixCode->errPos.nixFile != "") { + if (einfo.nixCode->errPos.file != "") { string eline = einfo.nixCode->errLineOfCode != "" ? string(" ") + showErrPos(einfo.nixCode->errPos) : ""; out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL, - prefix, - einfo.nixCode->errPos.nixFile, - eline) << std::endl; + prefix, + einfo.nixCode->errPos.file, + eline) << std::endl; out << prefix << std::endl; } else { out << fmt("%1%from command line argument", prefix) << std::endl; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0419a1b52..63778ebda 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -22,16 +22,16 @@ typedef enum { struct ErrPos { - int lineNumber; + int line; int column; - string nixFile; + string file; template ErrPos& operator=(const P &pos) { - lineNumber = pos.line; + line = pos.line; column = pos.column; - nixFile = pos.file; + file = pos.file; return *this; } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 98f500a65..4a8b98640 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -146,11 +146,40 @@ struct JSONLogger : Logger void logEI(const ErrorInfo & ei) override { - // nlohmann::json json; - // json["action"] = "msg"; - // json["level"] = lvl; - // json["msg"] = fs.s; - // write(json); + // add fields like Pos info and etc? + std::ostringstream oss; + oss << ei; + + nlohmann::json json; + json["action"] = "msg"; + json["level"] = ei.level; + json["msg"] = oss.str(); + + // Extra things that COULD go into json. Useful? + // TODO: decide if useful. + // TODO: make a json obj that goes into json["msg"]? + json["name"] = ei.name; + json["description"] = ei.description; + if (ei.hint.has_value()) + { + json["hint"] = ei.hint->str(); + } + if (ei.nixCode.has_value()) + { + if (ei.nixCode->errPos.line != 0) + json["line"] = ei.nixCode->errPos.line; + if (ei.nixCode->errPos.column != 0) + json["column"] = ei.nixCode->errPos.column; + if (ei.nixCode->errPos.file != "") + json["file"] = ei.nixCode->errPos.file; + if (ei.nixCode->prevLineOfCode.has_value()) + json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode; + json["errLineOfCode"] = ei.nixCode->errLineOfCode; + if (ei.nixCode->nextLineOfCode.has_value()) + json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode; + } + + write(json); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, From 15e9564fd1e11b3797f0188a494aa66d4622dfb0 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 19 Apr 2020 17:16:51 -0600 Subject: [PATCH 141/350] logEI for tunnelLogger and progressbar --- src/libstore/daemon.cc | 8 ++++++++ src/nix/progress-bar.cc | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index c8daef57c..d985f151b 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -75,6 +75,14 @@ struct TunnelLogger : public Logger void logEI(const ErrorInfo & ei) override { + if (ei.level > verbosity) return; + + std::stringstream oss; + oss << ei; + + StringSink buf; + buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + enqueueMsg(*buf.s); } /* startWork() means that we're starting an operation for which we diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 41f3d4cd7..b1226d550 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -127,7 +127,11 @@ public: void logEI(const ErrorInfo &ei) override { auto state(state_.lock()); - // log(*state, lvl, ei.as_str()); + + std::stringstream oss; + oss << ei; + + log(*state, ei.level, oss.str()); } void log(State & state, Verbosity lvl, const std::string & s) From d3052197feababc312fd874e08ae48050d985eb3 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 21 Apr 2020 13:25:41 -0600 Subject: [PATCH 142/350] add ErrorInfo to BaseError --- src/libutil/error.cc | 2 +- src/libutil/error.hh | 122 --------------------------------------- src/libutil/fmt.hh | 127 +++++++++++++++++++++++++++++++++++++++++ src/libutil/logging.hh | 1 - src/libutil/types.hh | 125 ++++++++++++++++++++-------------------- src/libutil/util.hh | 1 - 6 files changed, 192 insertions(+), 186 deletions(-) delete mode 100644 src/libutil/error.hh create mode 100644 src/libutil/fmt.hh diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 0e07c0d98..41595b0df 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,4 @@ -#include "error.hh" +#include "types.hh" #include #include diff --git a/src/libutil/error.hh b/src/libutil/error.hh deleted file mode 100644 index 63778ebda..000000000 --- a/src/libutil/error.hh +++ /dev/null @@ -1,122 +0,0 @@ -#ifndef error_hh -#define error_hh - -#include "ansicolor.hh" -#include -#include -#include -#include "types.hh" - -namespace nix -{ - -typedef enum { - lvlError = 0, - lvlWarn, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - -struct ErrPos -{ - int line; - int column; - string file; - - template - ErrPos& operator=(const P &pos) - { - line = pos.line; - column = pos.column; - file = pos.file; - return *this; - } - - template - ErrPos(const P &p) - { - *this = p; - } -}; - -struct NixCode -{ - ErrPos errPos; - std::optional prevLineOfCode; - string errLineOfCode; - std::optional nextLineOfCode; -}; - -// ---------------------------------------------------------------- -// format function for hints. same as fmt, except templated values -// are always in yellow. - -template -struct yellowify -{ - yellowify(T &s) : value(s) {} - T &value; -}; - -template -std::ostream& operator<<(std::ostream &out, const yellowify &y) -{ - return out << ANSI_YELLOW << y.value << ANSI_NORMAL; -} - -class hintformat -{ -public: - hintformat(string format) :fmt(format) - { - fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - } - template - hintformat& operator%(const T &value) - { - fmt % yellowify(value); - return *this; - } - - std::string str() const - { - return fmt.str(); - } - - template - friend class AddHint; -private: - format fmt; -}; - -std::ostream& operator<<(std::ostream &os, const hintformat &hf); - -template -inline hintformat hintfmt(const std::string & fs, const Args & ... args) -{ - hintformat f(fs); - formatHelper(f, args...); - return f; -} - -// ------------------------------------------------- -// ErrorInfo. -struct ErrorInfo -{ - Verbosity level; - string name; - string description; - std::optional hint; - std::optional nixCode; - - static std::optional programName; -}; - -std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); - -} - -#endif diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh new file mode 100644 index 000000000..08edffee0 --- /dev/null +++ b/src/libutil/fmt.hh @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include "ansicolor.hh" + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using boost::format; + + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template nop(T...) {} }; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + template + FormatOrString(const F & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + equivalent to ‘boost::format(format) % a_0 % ... % + ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + takes place). */ + +template +inline void formatHelper(F & f) +{ +} + +template +inline void formatHelper(F & f, const T & x, const Args & ... args) +{ + formatHelper(f % x, args...); +} + +inline std::string fmt(const std::string & s) +{ + return s; +} + +inline std::string fmt(const char * s) +{ + return s; +} + +inline std::string fmt(const FormatOrString & fs) +{ + return fs.s; +} + +template +inline std::string fmt(const std::string & fs, const Args & ... args) +{ + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + formatHelper(f, args...); + return f.str(); +} + +// ----------------------------------------------------------------------------- +// format function for hints in errors. same as fmt, except templated values +// are always in yellow. + +template +struct yellowify +{ + yellowify(T &s) : value(s) {} + T &value; +}; + +template +std::ostream& operator<<(std::ostream &out, const yellowify &y) +{ + return out << ANSI_YELLOW << y.value << ANSI_NORMAL; +} + +class hintformat +{ +public: + hintformat(const string &format) :fmt(format) + { + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + } + template + hintformat& operator%(const T &value) + { + fmt % yellowify(value); + return *this; + } + hintformat(const hintformat &hf) + : fmt(hf.fmt) + {} + + + std::string str() const + { + return fmt.str(); + } + + template + friend class AddHint; +private: + format fmt; +}; + +std::ostream& operator<<(std::ostream &os, const hintformat &hf); + +template +inline hintformat hintfmt(const std::string & fs, const Args & ... args) +{ + hintformat f(fs); + formatHelper(f, args...); + return f; +} + +} diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index ba6f39ac9..8269e2e1a 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,7 +1,6 @@ #pragma once #include "types.hh" -#include "error.hh" namespace nix { diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 981af528b..633c6bdf7 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -3,13 +3,13 @@ #include "ref.hh" -#include #include #include #include #include +#include -#include +#include "fmt.hh" /* Before 4.7, gcc's std::exception uses empty throw() specifiers for * its (virtual) destructor and what() in c++11 mode, in violation of spec @@ -20,73 +20,66 @@ #endif #endif - namespace nix { - -/* Inherit some names from other namespaces for convenience. */ -using std::string; using std::list; using std::set; using std::vector; -using boost::format; +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ -struct nop { template nop(T...) {} }; - - -struct FormatOrString +struct ErrPos { - string s; - FormatOrString(const string & s) : s(s) { }; - template - FormatOrString(const F & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; + int line; + int column; + string file; + + template + ErrPos& operator=(const P &pos) + { + line = pos.line; + column = pos.column; + file = pos.file; + return *this; + } + + template + ErrPos(const P &p) + { + *this = p; + } }; - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - -template -inline void formatHelper(F & f) +struct NixCode { -} + ErrPos errPos; + std::optional prevLineOfCode; + string errLineOfCode; + std::optional nextLineOfCode; +}; -template -inline void formatHelper(F & f, const T & x, const Args & ... args) +// ------------------------------------------------- +// ErrorInfo. +struct ErrorInfo { - formatHelper(f % x, args...); -} + Verbosity level; + string name; + string description; + std::optional hint; + std::optional nixCode; -inline std::string fmt(const std::string & s) -{ - return s; -} - -inline std::string fmt(const char * s) -{ - return s; -} - -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - -template -inline std::string fmt(const std::string & fs, const Args & ... args) -{ - boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - formatHelper(f, args...); - return f.str(); -} + static std::optional programName; +}; +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); /* BaseError should generally not be caught, as it has Interrupted as a subclass. Catch Error instead. */ @@ -94,33 +87,42 @@ class BaseError : public std::exception { protected: string prefix_; // used for location traces etc. - string err; + ErrorInfo err; public: unsigned int status = 1; // exit status template BaseError(unsigned int status, const Args & ... args) - : err(fmt(args...)) + : err(hintfmt(args...)) , status(status) { } template BaseError(const Args & ... args) - : err(fmt(args...)) + : err { .level = lvlError, + .hint = hintfmt(args...) + } + { + } + + BaseError(ErrorInfo e) + : err(e) { } #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; - const char * what() const throw () { return err.c_str(); } + const char * what() const throw () { return err.description.c_str(); } #else - const char * what() const noexcept { return err.c_str(); } + const char * what() const noexcept { return err.description.c_str(); } #endif - const string & msg() const { return err; } + const string & msg() const { return err.description; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); + + const ErrorInfo & info() const { return err; } }; #define MakeError(newClass, superClass) \ @@ -139,7 +141,8 @@ public: template SysError(const Args & ... args) - : Error(addErrno(fmt(args...))) + : Error(args...) // TODO addErrNo for hintfmt + // : Error(addErrno(hintfmt(args...))) { } private: diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fb9a7ed38..636f3ba6c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "logging.hh" #include "ansicolor.hh" -#include "error.hh" #include #include From e4fb9a38493a041861fe5c75bc8ddd129a2e5262 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 21 Apr 2020 17:07:07 -0600 Subject: [PATCH 143/350] remove 'format' from Error constructor calls --- src/libexpr/attr-path.cc | 16 +-- src/libexpr/eval-inline.hh | 6 +- src/libexpr/eval.cc | 23 ++--- src/libexpr/nixexpr.cc | 2 +- src/libexpr/nixexpr.hh | 4 +- src/libexpr/parser.y | 32 +++--- src/libexpr/primops.cc | 124 +++++++++++------------ src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/value-to-json.cc | 4 +- src/libmain/shared.cc | 10 +- src/libmain/shared.hh | 6 +- src/libstore/binary-cache-store.cc | 8 +- src/libstore/build.cc | 98 +++++++++--------- src/libstore/builtins/buildenv.cc | 10 +- src/libstore/builtins/fetchurl.cc | 4 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 6 +- src/libstore/download.cc | 6 +- src/libstore/download.hh | 6 +- src/libstore/gc.cc | 44 ++++---- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-fs-store.cc | 6 +- src/libstore/local-store.cc | 48 ++++----- src/libstore/nar-accessor.cc | 6 +- src/libstore/nar-info.cc | 2 +- src/libstore/optimise-store.cc | 24 ++--- src/libstore/pathlocks.cc | 8 +- src/libstore/profiles.cc | 8 +- src/libstore/references.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-store.cc | 4 +- src/libstore/s3-binary-cache-store.cc | 8 +- src/libstore/sqlite.cc | 2 +- src/libstore/store-api.cc | 4 +- src/libutil/archive.cc | 19 ++-- src/libutil/args.cc | 9 +- src/libutil/compression.cc | 2 +- src/libutil/fmt.hh | 6 +- src/libutil/types.hh | 4 +- src/libutil/util.cc | 50 ++++----- src/nix-channel/nix-channel.cc | 30 ++++-- src/nix-daemon/nix-daemon.cc | 43 ++++---- src/nix-env/nix-env.cc | 55 +++++----- src/nix-instantiate/nix-instantiate.cc | 4 +- src/nix-prefetch-url/nix-prefetch-url.cc | 10 +- src/nix-store/nix-store.cc | 38 +++---- src/nix/cat.cc | 4 +- src/nix/hash.cc | 2 +- src/nix/ls.cc | 2 +- src/nix/repl.cc | 8 +- src/nix/run.cc | 4 +- 52 files changed, 424 insertions(+), 407 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 76d101b98..9a9531a3f 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s) ++i; while (1) { if (i == s.end()) - throw Error(format("missing closing quote in selection path '%1%'") % s); + throw Error("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -60,11 +60,11 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (v->type != tAttrs) throw TypeError( - format("the expression selected by the selection path '%1%' should be a set but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a set but is %2%", + attrPath, + showType(*v)); if (attr.empty()) - throw Error(format("empty attribute name in selection path '%1%'") % attrPath); + throw Error("empty attribute name in selection path '%1%'", attrPath); Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) @@ -77,9 +77,9 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr if (!v->isList()) throw TypeError( - format("the expression selected by the selection path '%1%' should be a list but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a list but is %2%", + attrPath, + showType(*v)); if (attrIndex >= v->listSize()) throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c27116e3b..4d82ccf09 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -9,18 +9,18 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) { - throw EvalError(format(s) % pos); + throw EvalError(s, pos); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) { - throw TypeError(format(s) % showType(v)); + throw TypeError(s, showType(v)); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) { - throw TypeError(format(s) % showType(v) % pos); + throw TypeError(s, showType(v), pos); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..516c25b02 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -493,52 +493,52 @@ Value & EvalState::getBuiltin(const string & name) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) { - throw EvalError(format(s) % s2); + throw EvalError(s, s2); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) { - throw EvalError(format(s) % s2 % pos); + throw EvalError(s, s2, pos); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) { - throw EvalError(format(s) % s2 % s3); + throw EvalError(s, s2, s3); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) { - throw EvalError(format(s) % s2 % s3 % pos); + throw EvalError(s, s2, s3, pos); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) { - throw EvalError(format(s) % sym % p1 % p2); + throw EvalError(s, sym, p1, p2); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) { - throw TypeError(format(s) % pos); + throw TypeError(s, pos); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) { - throw TypeError(format(s) % s1); + throw TypeError(s, s1); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) { - throw TypeError(format(s) % fun.showNamePos() % s2 % pos); + throw TypeError(s, fun.showNamePos(), s2, pos); } LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) { - throw AssertionError(format(s) % s1 % pos); + throw AssertionError(s, s1, pos); } LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) { - throw UndefinedVarError(format(s) % s1 % pos); + throw UndefinedVarError(s, s1, pos); } LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) @@ -1883,8 +1883,7 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { - throw TypeError(format("cannot coerce %1% to a string, at %2%") % - showType() % pos); + throw TypeError("cannot coerce %1% to a string, at %2%", showType(), pos); } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 63cbef1dd..a9955f6df 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -267,7 +267,7 @@ void ExprVar::bindVars(const StaticEnv & env) /* Otherwise, the variable must be obtained from the nearest enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); + if (withLevel == -1) throw UndefinedVarError("undefined variable '%1%' at %2%", name, pos); fromWith = true; this->level = withLevel; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index f7e9105a4..79e3f90e5 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -234,9 +234,7 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument '%1%' at %2%") - % arg % pos); - }; + throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); }; void setName(Symbol & name); string showNamePos() const; COMMON_METHODS diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..3767532d5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,15 +64,15 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute '%1%' at %2% already defined at %3%") - % showAttrPath(attrPath) % pos % prevPos); + throw ParseError("attribute '%1%' at %2% already defined at %3%", + showAttrPath(attrPath), pos, prevPos); } static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute '%1%' at %2% already defined at %3%") - % attr % pos % prevPos); + throw ParseError("attribute '%1%' at %2% already defined at %3%", + attr, pos, prevPos); } @@ -140,8 +140,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) - throw ParseError(format("duplicate formal function argument '%1%' at %2%") - % formal.name % pos); + throw ParseError("duplicate formal function argument '%1%' at %2%", + formal.name, pos); formals->formals.push_front(formal); } @@ -327,8 +327,7 @@ expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) - throw ParseError(format("dynamic attributes not allowed in let at %1%") - % CUR_POS); + throw ParseError("dynamic attributes not allowed in let at %1%", CUR_POS); $$ = new ExprLet($2, $4); } | expr_if @@ -475,8 +474,8 @@ attrs $$->push_back(AttrName(str->s)); delete str; } else - throw ParseError(format("dynamic attributes not allowed in inherit at %1%") - % makeCurPos(@2, data)); + throw ParseError("dynamic attributes not allowed in inherit at %1%", + makeCurPos(@2, data)); } | { $$ = new AttrPath; } ; @@ -670,11 +669,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - format f = format( + string f = "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)" - + string(pos ? ", at %2%" : "")); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - throw ThrownError(f % path % pos); + + string(pos ? ", at %2%" : ""); + throw ThrownError(f, path, pos); } @@ -691,7 +689,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl request.unpack = true; res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { - printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); + // TODO: change to warn()? + printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second); res = { false, "" }; } } else { @@ -699,7 +698,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (pathExists(path)) res = { true, path }; else { - printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second); + // TODO: change to warn()? + printError("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second); res = { false, "" }; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..13eb1ba58 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -94,8 +94,8 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%", + path, e.path, pos); } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -171,8 +171,8 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%" + , path, e.path, pos); } path = state.checkSourcePath(path); @@ -181,17 +181,17 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw EvalError(format("could not open '%1%': %2%") % path % dlerror()); + throw EvalError("could not open '%1%': %2%", path, dlerror()); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message); + throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message); else - throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected") - % sym % path); + throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected" + , sym, path); } (func)(state, v); @@ -207,7 +207,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) { - throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos); + throw EvalError("at least one argument to 'exec' required, at %1%", pos); } PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); @@ -218,8 +218,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%") - % program % e.path % pos); + throw EvalError("cannot execute '%1%', since path '%2%' is not valid, at %3%" + , program, e.path, pos); } auto output = runProgram(program, true, commandArgs); @@ -227,13 +227,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { parsed = state.parseExprFromString(output, pos.file); } catch (Error & e) { - e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos); + e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos)); throw; } try { state.eval(parsed, v); } catch (Error & e) { - e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos); + e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos)); throw; } } @@ -339,7 +339,7 @@ struct CompareValues if (v1->type == tInt && v2->type == tFloat) return v1->integer < v2->fpoint; if (v1->type != v2->type) - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); switch (v1->type) { case tInt: return v1->integer < v2->integer; @@ -350,7 +350,7 @@ struct CompareValues case tPath: return strcmp(v1->path, v2->path) < 0; default: - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } } }; @@ -371,7 +371,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator startSet = args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) - throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + throw EvalError("attribute 'startSet' required, at %1%", pos); state.forceList(*startSet->value, pos); ValueList workSet; @@ -382,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator op = args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) - throw EvalError(format("attribute 'operator' required, at %1%") % pos); + throw EvalError("attribute 'operator' required, at %1%", pos); state.forceValue(*op->value); /* Construct the closure by applying the operator to element of @@ -401,7 +401,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator key = e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) - throw EvalError(format("attribute 'key' required, at %1%") % pos); + throw EvalError("attribute 'key' required, at %1%", pos); state.forceValue(*key->value); if (!doneKeys.insert(key->value).second) continue; @@ -431,7 +431,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; string s = state.coerceToString(pos, *args[0], context); - throw Abort(format("evaluation aborted with the following error message: '%1%'") % s); + throw Abort("evaluation aborted with the following error message: '%1%'", s); } @@ -506,9 +506,9 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value { state.forceValue(*args[0]); if (args[0]->type == tString) - printError(format("trace: %1%") % args[0]->string.s); + printError("trace: %1%", args[0]->string.s); else - printError(format("trace: %1%") % *args[0]); + printError("trace: %1%", *args[0]); state.forceValue(*args[1]); v = *args[1]; } @@ -533,13 +533,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Figure out the name first (for stack backtraces). */ Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) - throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + throw EvalError("required attribute 'name' missing, at %1%", pos); string drvName; Pos & posDrvName(*attr->pos); try { drvName = state.forceStringNoCtx(*attr->value, pos); } catch (Error & e) { - e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName); + e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName)); throw; } @@ -583,18 +583,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName); + throw EvalError("duplicate derivation output '%1%', at %2%", j, posDrvName); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName); + throw EvalError("invalid derivation output name 'drv', at %1%", posDrvName); outputs.insert(j); } if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + throw EvalError("derivation cannot have an empty set of outputs, at %1%", posDrvName); }; try { @@ -706,9 +706,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName); + throw EvalError("required attribute 'builder' missing, at %1%", posDrvName); if (drv.platform == "") - throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName); + throw EvalError("required attribute 'system' missing, at %1%", posDrvName); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) @@ -717,7 +717,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); + throw Error("multiple outputs are not supported in fixed-output derivations, at %1%", posDrvName); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); @@ -818,7 +818,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) - throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos); + throw EvalError("path '%1%' is not in the Nix store, at %2%", path, pos); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(path2)); @@ -834,9 +834,9 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format( - "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%", + path, e.path, pos); } try { @@ -879,12 +879,12 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%" + , path, e.path, pos); } string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) - throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path); + throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); mkString(v, s.c_str()); } @@ -908,7 +908,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) - throw EvalError(format("attribute 'path' missing, at %1%") % pos); + throw EvalError("attribute 'path' missing, at %1%", pos); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -916,8 +916,8 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError("cannot find '%1%', since path '%2%' is not valid, at %3%", + path, e.path, pos); } searchPath.emplace_back(prefix, path); @@ -934,7 +934,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + throw Error("unknown hash type '%1%', at %2%", type, pos); PathSet context; // discarded Path p = state.coerceToPath(pos, *args[1], context); @@ -950,8 +950,8 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val try { state.realiseContext(ctx); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%", + path, e.path, pos); } DirEntries entries = readDirectory(state.checkSourcePath(path)); @@ -1021,7 +1021,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); + throw EvalError("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%", name, pos); refs.insert(state.store->parseStorePath(path)); } @@ -1089,11 +1089,11 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, pos); state.forceValue(*args[0]); if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + throw TypeError("first argument in call to 'filterSource' is not a function but %1%, at %2%", showType(*args[0]), pos); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); } @@ -1113,7 +1113,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value PathSet context; path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, *attr.pos); } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { @@ -1124,10 +1124,10 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else - throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + throw EvalError("unsupported argument '%1%' to 'addPath', at %2%", attr.name, *attr.pos); } if (path.empty()) - throw EvalError(format("'path' required, at %1%") % pos); + throw EvalError("'path' required, at %1%", pos); if (name.empty()) name = baseNameOf(path); @@ -1185,7 +1185,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + throw EvalError("attribute '%1%' missing, at %2%", attr, pos); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value); @@ -1265,14 +1265,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) - throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos); + throw TypeError("'name' attribute missing in a call to 'listToAttrs', at %1%", pos); string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) - throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); + throw TypeError("'value' attribute missing in a call to 'listToAttrs', at %1%", pos); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } @@ -1346,7 +1346,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args { state.forceValue(*args[0]); if (args[0]->type != tLambda) - throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); + throw TypeError("'functionArgs' requires a function, at %1%", pos); if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); @@ -1396,7 +1396,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); + throw Error("list index %1% is out of bounds, at %2%", n, pos); state.forceValue(*list.listElems()[n]); v = *list.listElems()[n]; } @@ -1423,7 +1423,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value { state.forceList(*args[0], pos); if (args[0]->listSize() == 0) - throw Error(format("'tail' called on an empty list, at %1%") % pos); + throw Error("'tail' called on an empty list, at %1%", pos); state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) v.listElems()[n] = args[0]->listElems()[n + 1]; @@ -1564,7 +1564,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val auto len = state.forceInt(*args[1], pos); if (len < 0) - throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + throw EvalError("cannot create list of size %1%, at %2%", len, pos); state.mkList(v, len); @@ -1722,7 +1722,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[1], pos); NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + if (f2 == 0) throw EvalError("division by zero, at %1%", pos); if (args[0]->type == tFloat || args[1]->type == tFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); @@ -1731,7 +1731,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixInt i2 = state.forceInt(*args[1], pos); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - throw EvalError(format("overflow in integer division, at %1%") % pos); + throw EvalError("overflow in integer division, at %1%", pos); mkInt(v, i1 / i2); } } @@ -1787,7 +1787,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V PathSet context; string s = state.coerceToString(pos, *args[2], context); - if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos); + if (start < 0) throw EvalError("negative start position in 'substring', at %1%", pos); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } @@ -1807,7 +1807,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + throw Error("unknown hash type '%1%', at %2%", type, pos); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); @@ -1950,7 +1950,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[0], pos); state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos); + throw EvalError("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%", pos); vector from; from.reserve(args[0]->listSize()); @@ -2072,11 +2072,11 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, else if (n == "name") request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); + throw EvalError("unsupported argument '%1%' to '%2%', at %3%", attr.name, who, attr.pos); } if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + throw EvalError("'url' argument required, at %1%", pos); } else request.uri = state.forceStringNoCtx(*args[0], pos); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..480cc1adc 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -221,7 +221,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va } if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + throw EvalError("'url' argument required, at %1%", pos); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..76029f548 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -191,7 +191,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar } if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + throw EvalError("'url' argument required, at %1%", pos); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5fe8570ad..6ec8315ba 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; default: - throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + throw TypeError("cannot convert %1% to JSON", showType(v)); } } @@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, JSONPlaceholder & out, PathSet & context) const { - throw TypeError(format("cannot convert %1% to JSON") % showType()); + throw TypeError("cannot convert %1% to JSON", showType()); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..f7ab56271 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -75,7 +75,7 @@ string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); return *i; } @@ -229,7 +229,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish) Strings ss(args); auto pos = ss.begin(); if (!parseArg(pos, ss.end())) - throw UsageError(format("unexpected argument '%1%'") % args.front()); + throw UsageError("unexpected argument '%1%'", args.front()); return true; } @@ -273,7 +273,7 @@ void showManPage(const string & name) restoreSignals(); setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), nullptr); - throw SysError(format("command 'man %1%' failed") % name.c_str()); + throw SysError("command 'man %1%' failed", name.c_str()); } @@ -301,7 +301,7 @@ int handleExceptions(const string & programName, std::function fun) % e.what() % programName); return 1; } catch (BaseError & e) { - printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printError(error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); if (e.prefix() != "" && !settings.showTrace) printError("(use '--show-trace' to show detailed location information)"); return e.status; @@ -338,7 +338,7 @@ RunPager::RunPager() execlp("pager", "pager", nullptr); execlp("less", "less", nullptr); execlp("more", "more", nullptr); - throw SysError(format("executing '%1%'") % pager); + throw SysError("executing '%1%'", pager); }); pid.setKillSignal(SIGINT); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index b49574652..f558247c0 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -56,7 +56,7 @@ template N getIntArg(const string & opt, Strings::iterator & i, const Strings::iterator & end, bool allowUnit) { ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); string s = *i; N multiplier = 1; if (allowUnit && !s.empty()) { @@ -66,13 +66,13 @@ template N getIntArg(const string & opt, else if (u == 'M') multiplier = 1ULL << 20; else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError(format("invalid unit specifier '%1%'") % u); + else throw UsageError("invalid unit specifier '%1%'", u); s.resize(s.size() - 1); } } N n; if (!string2Int(s, n)) - throw UsageError(format("'%1%' requires an integer argument") % opt); + throw UsageError("'%1%' requires an integer argument", opt); return n * multiplier; } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a2d84861..4b7385c6b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -40,14 +40,14 @@ void BinaryCacheStore::init() upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; + size_t colon= line.find(':'); + if (colon ==std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") - % getUri() % value % storeDir); + throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", + getUri(), value, storeDir); } else if (name == "WantMassQuery") { wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 527d7ac42..2301adb16 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -453,7 +453,7 @@ static void commonChildInit(Pipe & logPipe) that e.g. ssh cannot open /dev/tty) and it doesn't receive terminal signals. */ if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) @@ -466,7 +466,7 @@ static void commonChildInit(Pipe & logPipe) /* Reroute stdin to /dev/null. */ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); if (fdDevNull == -1) - throw SysError(format("cannot open '%1%'") % pathNullDevice); + throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) throw SysError("cannot dup null device into stdin"); close(fdDevNull); @@ -488,7 +488,7 @@ void handleDiffHook( auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); @@ -534,8 +534,8 @@ UserLock::UserLock() /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + throw Error("the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ @@ -546,8 +546,8 @@ UserLock::UserLock() } if (users.empty()) - throw Error(format("the build users group '%1%' has no members") - % settings.buildUsersGroup); + throw Error("the build users group '%1%' has no members", + settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ @@ -556,8 +556,8 @@ UserLock::UserLock() struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error(format("the user '%1%' in the group '%2%' does not exist") - % i % settings.buildUsersGroup); + throw Error("the user '%1%' in the group '%2%' does not exist", + i, settings.buildUsersGroup); createDirs(settings.nixStateDir + "/userpool"); @@ -565,7 +565,7 @@ UserLock::UserLock() AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); + throw SysError("opening user lock '%1%'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { fdUserLock = std::move(fd); @@ -574,8 +574,8 @@ UserLock::UserLock() /* Sanity check... */ if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%1%'", + settings.buildUsersGroup); #if __linux__ /* Get the list of supplementary groups of this build user. This @@ -585,7 +585,7 @@ UserLock::UserLock() int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), &ngroups); if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); supplementaryGIDs.resize(ngroups); #endif @@ -594,9 +594,9 @@ UserLock::UserLock() } } - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the '%1%' group") - % settings.buildUsersGroup); + throw Error("all build users are currently in use; " + "consider creating additional users and adding them to the '%1%' group", + settings.buildUsersGroup); } @@ -1985,7 +1985,7 @@ void DerivationGoal::startBuilder() string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString(s); if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); @@ -2034,7 +2034,7 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + throw Error("while processing 'sandbox-paths': %s", e.what()); } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2081,10 +2081,10 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create '%1%'") % chrootRootDir); + throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); + throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR @@ -2128,7 +2128,7 @@ void DerivationGoal::startBuilder() chmod_(chrootStoreDir, 01775); if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); @@ -2161,7 +2161,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); + throw Error("directory '%1%' exists; please remove it", homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete @@ -2206,8 +2206,7 @@ void DerivationGoal::startBuilder() if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { - throw Error(format("unknown pre-build hook command '%1%'") - % line); + throw Error("unknown pre-build hook command '%1%'", line); } } else if (state == stExtraChrootDirs) { if (line == "") { @@ -2967,7 +2966,7 @@ void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); + throw SysError("cannot change ownership of '%1%'", path); } @@ -3104,7 +3103,7 @@ void DerivationGoal::runChild() /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + throw SysError("unable to bind mount '%1%'", chrootRootDir); /* Bind-mount the sandbox's Nix store onto itself so that we can mark it as a "shared" subtree, allowing bind @@ -3238,16 +3237,16 @@ void DerivationGoal::runChild() /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); + throw SysError("cannot change directory to '%1%'", chrootRootDir); if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); if (chroot(".") == -1) - throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + throw SysError("cannot change root directory to '%1%'", chrootRootDir); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -3268,7 +3267,7 @@ void DerivationGoal::runChild() #endif if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into '%1%'") % tmpDir); + throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); @@ -3407,9 +3406,9 @@ void DerivationGoal::runChild() sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); string path = i.first; struct stat st; @@ -3500,7 +3499,7 @@ void DerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); @@ -3510,7 +3509,7 @@ void DerivationGoal::runChild() execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing '%1%'") % drv->builder); + throw SysError("executing '%1%'", drv->builder); } catch (std::exception & e) { writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); @@ -3595,7 +3594,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); + throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } @@ -3616,13 +3615,13 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); + throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); #endif /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); + printError("warning: rewriting hashes in '%1%'; cross fingers", path); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -3654,7 +3653,8 @@ void DerivationGoal::registerOutputs() /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); + "output path '%1%' should be a non-executable regular file", + path); } /* Check the hash. In hash mode, move the path produced by @@ -3715,7 +3715,7 @@ void DerivationGoal::registerOutputs() Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + throw SysError("renaming '%1%' to '%2%'", actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), @@ -4014,7 +4014,7 @@ Path DerivationGoal::openLogFile() settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared(fdLogFile.get()); @@ -4522,7 +4522,6 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -4739,9 +4738,9 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( - "unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); assert(!awake.empty()); } } @@ -4754,7 +4753,6 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || children.empty()); } - void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); @@ -4895,6 +4893,9 @@ void Worker::waitForInput() } + + + unsigned int Worker::exitStatus() { /* @@ -4994,7 +4995,6 @@ void LocalStore::buildPaths(const std::vector & drvPaths, throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); } - BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { @@ -5055,4 +5055,8 @@ void LocalStore::repairPath(const StorePath & path) } + } + + + diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 1b802d908..000769094 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -72,15 +72,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (!S_ISDIR(lstat(target).st_mode)) throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); + throw SysError("creating directory '%1%'", dstFile); createLinks(state, target, dstFile, state.priorities[dstFile]); createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } else { @@ -99,11 +99,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } createSymlink(srcFile, dstFile); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..223ed1fba 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto getAttr = [&](const string & name) { auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; @@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto executable = drv.env.find("executable"); if (executable != drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); + throw SysError("making '%1%' executable", storePath); } }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d985f151b..f8b449e5e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -747,7 +747,7 @@ static void performOp(TunnelLogger * logger, ref store, } default: - throw Error(format("invalid operation %1%") % op); + throw Error("invalid operation %1%", op); } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ddc86a..f35b8cc43 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -87,7 +87,7 @@ static void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw FormatError(format("expected string '%1%'") % s); + throw FormatError("expected string '%1%'", s); } @@ -114,7 +114,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path '%1%' in derivation") % s); + throw FormatError("bad path '%1%' in derivation", s); return s; } @@ -196,7 +196,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath) try { return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5967d0425..7d0cb449b 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -113,7 +113,7 @@ struct CurlDownloader : public Downloader if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(DownloadError(Interrupted, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -518,7 +518,7 @@ struct CurlDownloader : public Downloader int running; CURLMcode mc = curl_multi_perform(curlm, &running); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); /* Set the promises of any finished requests. */ CURLMsg * msg; @@ -548,7 +548,7 @@ struct CurlDownloader : public Downloader vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); nextWakeup = std::chrono::steady_clock::time_point(); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 5a131c704..8141928d2 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -128,8 +128,10 @@ class DownloadError : public Error { public: Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) - : Error(fs), error(error) + + template + DownloadError(Downloader::Error error, const Args & ... args) + : Error(args...), error(error) { } }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0c3d89611..27bd3f3e9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -38,7 +38,7 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fdGCLock) - throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { printError(format("waiting for the big garbage collector lock...")); @@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target) /* Atomically replace the old one. */ if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename '%1%' to '%2%'") - % tempLink % link); + throw SysError("cannot rename '%1%' to '%2%'", + tempLink , link); } @@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path gcRoot(canonPath(_gcRoot)); if (isInStore(gcRoot)) - throw Error(format( + throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); + "(are you running nix-build inside the store?)", gcRoot); if (indirect) { /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); + throw Error("cannot create symlink '%1%'; already exists", gcRoot); makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( + throw Error( "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'") - % gcRoot % rootsDir); + "it's not in the directory '%2%'", + gcRoot, rootsDir); } if (baseNameOf(gcRoot) == std::string(storePath.to_string())) @@ -170,7 +170,7 @@ void LocalStore::addTempRoot(const StorePath & path) way. */ struct stat st; if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting '%1%'") % fnTempRoots); + throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -211,7 +211,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) if (!*fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file '%1%'") % path); + throw SysError("opening temporary roots file '%1%'", path); } /* This should work, but doesn't, for some reason. */ @@ -222,7 +222,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); + printError("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -398,7 +398,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!fdDir) { if (errno == ENOENT || errno == EACCES) continue; - throw SysError(format("opening %1%") % fdStr); + throw SysError("opening %1%", fdStr); } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { @@ -408,7 +408,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (errno) { if (errno == ESRCH) continue; - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + throw SysError("iterating /proc/%1%/fd", ent->d_name); } fdDir.reset(); @@ -536,7 +536,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); + throw SysError("getting status of %1%", realPath); } printInfo(format("deleting '%1%'") % path); @@ -554,10 +554,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) // size. try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making '%1%' writable") % realPath); + throw SysError("making '%1%' writable", realPath); Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); + throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); state.bytesInvalidated += size; } catch (SysError & e) { if (e.errNo == ENOSPC) { @@ -676,7 +676,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) void LocalStore::removeUnusedLinks(const GCState & state) { AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); long long actualSize = 0, unsharedSize = 0; @@ -689,7 +689,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % path); + throw SysError("statting '%1%'", path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -700,14 +700,14 @@ void LocalStore::removeUnusedLinks(const GCState & state) printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) - throw SysError(format("deleting '%1%'") % path); + throw SysError("deleting '%1%'", path); state.results.bytesFreed += st.st_size; } struct stat st; if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % linksDir); + throw SysError("statting '%1%'", linksDir); long long overhead = st.st_blocks * 512ULL; printInfo(format("note: currently hard linking saves %.2f MiB") @@ -801,7 +801,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) try { AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); + if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 363be1443..48aca478c 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s) AutoDelete del(tmp, false); writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + throw SysError("renaming '%1%' to '%2%'", tmp, path); del.cancel(); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index aa5abd835..2d564a0d7 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor { Path storePath = store->toStorePath(path); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file '%1%' has unsupported type") % path); + throw Error("file '%1%' has unsupported type", path); return { S_ISREG(st.st_mode) ? Type::tRegular : diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ae7513ad8..db5a3b53c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -92,13 +92,13 @@ LocalStore::LocalStore(const Params & params) else { struct stat st; if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % realStoreDir); + throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path '%1%'") % realStoreDir); + throw SysError("changing ownership of path '%1%'", realStoreDir); if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path '%1%'") % realStoreDir); + throw SysError("changing permissions on path '%1%'", realStoreDir); } } } @@ -109,12 +109,12 @@ LocalStore::LocalStore(const Params & params) struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); if (S_ISLNK(st.st_mode)) - throw Error(format( + throw Error( "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); + "this is not allowed for the Nix store and its parent directories", + path); path = dirOf(path); } } @@ -155,8 +155,8 @@ LocalStore::LocalStore(const Params & params) upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); + throw Error("current Nix store schema is version %1%, but I only support %2%", + curSchema, nixSchemaVersion); else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; @@ -284,7 +284,7 @@ int LocalStore::getSchema() if (pathExists(schemaPath)) { string s = readFile(schemaPath); if (!string2Int(s, curSchema)) - throw Error(format("'%1%' is corrupt") % schemaPath); + throw Error("'%1%' is corrupt", schemaPath); } return curSchema; } @@ -293,7 +293,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); + throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; @@ -367,7 +367,7 @@ void LocalStore::makeStoreWritable() throw SysError("setting up a private mount namespace"); if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); + throw SysError("remounting %1% writable", realStoreDir); } #endif } @@ -388,7 +388,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + throw SysError("changing mode of '%1%' to %2$o", path, mode); } } @@ -406,7 +406,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of '%1%'") % path); + throw SysError("changing modification time of '%1%'", path); } } @@ -415,7 +415,7 @@ void canonicaliseTimestampAndPermissions(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); canonicaliseTimestampAndPermissions(path, st); } @@ -430,17 +430,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe setattrlist() to remove other attributes as well. */ if (lchflags(path.c_str(), 0)) { if (errno != ENOTSUP) - throw SysError(format("clearing flags of path '%1%'") % path); + throw SysError("clearing flags of path '%1%'", path); } #endif struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file '%1%' has an unsupported type") % path); + throw Error("file '%1%' has an unsupported type", path); #if __linux__ /* Remove extended attributes / ACLs. */ @@ -474,7 +474,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file '%1%'") % path); + throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; @@ -498,8 +498,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of '%1%' to %2%") - % path % geteuid()); + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); } if (S_ISDIR(st.st_mode)) { @@ -518,11 +518,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); + throw Error("wrong ownership of top-level store path '%1%'", path); } } @@ -1392,7 +1392,7 @@ static void makeMutable(const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) { if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); } unsigned int flags = 0, old; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b74480684..ca663d837 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor auto i = get(path); if (i.type != FSAccessor::Type::tDirectory) - throw Error(format("path '%1%' inside NAR file is not a directory") % path); + throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; for (auto & child : i.children) @@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.start, i.size); @@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tSymlink) - throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..fbdcd7786 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,7 +7,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack { auto corrupt = [&]() { - throw Error(format("NAR info file '%1%' is corrupt") % whence); + throw Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..f9cf2bb6b 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -19,9 +19,9 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); + throw SysError("changing writability of '%1%'", path); } @@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() InodeHash inodeHash; AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + if (errno) throw SysError("reading directory '%1%'", linksDir); printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); @@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa Strings names; AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa if (name == "." || name == "..") continue; names.push_back(name); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return names; } @@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -130,7 +130,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); + printError("skipping suspicious writable file '%1%'", path); return; } @@ -186,7 +186,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); + throw SysError("getting attributes of path '%1%'", linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -194,7 +194,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); + printError("removing corrupted link '%1%'", linkPath); unlink(linkPath.c_str()); goto retry; } @@ -229,7 +229,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); + printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it @@ -238,7 +238,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); } stats.filesLinked++; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 2635e3940..52d430ffd 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create) fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file '%1%'") % path); + throw SysError("opening lock file '%1%'", path); return fd; } @@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait) while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); else return false; } @@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait) checkInterrupt(); if (errno == EWOULDBLOCK) return false; if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); } } @@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths, hasn't been unlinked). */ struct stat st; if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); + throw SysError("statting lock file '%1%'", lockPath); if (st.st_size != 0) /* This lock file has been unlinked, so we're holding a lock on a deleted file. This means that other diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 2bef51878..6cfe393a4 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen) gen.number = n; struct stat st; if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting '%1%'") % gen.path); + throw SysError("statting '%1%'", gen.path); gen.creationTime = st.st_mtime; gens.push_back(gen); } @@ -117,7 +117,7 @@ Path createGeneration(ref store, Path profile, Path outPath) static void removeFile(const Path & path) { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } @@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set & gens Generations gens = findGenerations(profile, curGen); if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%'") % profile); + throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { if (gensToDelete.find(i.number) == gensToDelete.end()) continue; @@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b int days; if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + throw Error("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - days * 24 * 3600; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 102e15921..a10d536a3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path, auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference '%1%'") % i); + throw Error("bad reference '%1%'", i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5a2d103b9..9277a8e6b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -51,7 +51,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) std::string restPath = std::string(path, storePath.size()); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); auto i = nars.find(storePath); if (i != nars.end()) return {i->second, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..0f8126aee 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -116,11 +116,11 @@ ref UDSRemoteStore::openConnection() struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPath); + throw Error("socket path '%1%' is too long", socketPath); strcpy(addr.sun_path, socketPath.c_str()); if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + throw SysError("cannot connect to daemon at '%1%'", socketPath); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..0326821f6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -32,8 +32,10 @@ namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; + + template + S3Error(Aws::S3::S3Errors err, const Args & ... args) + : Error(args...), err(err) { }; }; /* Helper: given an Outcome, return R in case of success, or @@ -249,7 +251,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore // If bucket listing is disabled, 404s turn into 403s || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; - throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); + throw Error("AWS error fetching '%s': %s", path, error.GetMessage()); } return true; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index eb1daafc5..3407e5826 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -29,7 +29,7 @@ SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database '%s'") % path); + throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throwSQLiteError(db, "setting timeout"); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b9e894a9a..a04d5013e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -22,7 +22,7 @@ bool Store::isInStore(const Path & path) const Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return path; @@ -40,7 +40,7 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); return path; } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212..6a8484705 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -46,7 +46,7 @@ static void dumpContents(const Path & path, size_t size, sink << "contents" << size; AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); + if (!fd) throw SysError("opening file '%1%'", path); std::vector buf(65536); size_t left = size; @@ -68,7 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); sink << "("; @@ -94,8 +94,9 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) name.erase(pos); } if (unhacked.find(name) != unhacked.end()) - throw Error(format("file name collision in between '%1%' and '%2%'") - % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + throw Error("file name collision in between '%1%' and '%2%'", + (path + "/" + unhacked[name]), + (path + "/" + i.name)); unhacked[name] = i.name; } else unhacked[i.name] = i.name; @@ -111,7 +112,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else if (S_ISLNK(st.st_mode)) sink << "type" << "symlink" << "target" << readLink(path); - else throw Error(format("file '%1%' has an unsupported type") % path); + else throw Error("file '%1%' has an unsupported type", path); sink << ")"; } @@ -247,7 +248,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "name") { name = readString(source); if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) - throw Error(format("NAR contains invalid file name '%1%'") % name); + throw Error("NAR contains invalid file name '%1%'", name); if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; @@ -303,14 +304,14 @@ struct RestoreSink : ParseSink { Path p = dstPath + path; if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); + throw SysError("creating directory '%1%'", p); }; void createRegularFile(const Path & path) { Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); + if (!fd) throw SysError("creating file '%1%'", p); } void isExecutable() @@ -332,7 +333,7 @@ struct RestoreSink : ParseSink OpenSolaris). Since preallocation is just an optimisation, ignore it. */ if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); + throw SysError("preallocating file of %1% bytes", len); } #endif } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ba15ea571..4e705dd64 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -47,7 +47,7 @@ void Args::parseCmdline(const Strings & _cmdline) } else if (!dashDash && std::string(arg, 0, 1) == "-") { if (!processFlag(pos, cmdline.end())) - throw UsageError(format("unrecognised flag '%1%'") % arg); + throw UsageError("unrecognised flag '%1%'", arg); } else { pendingArgs.push_back(*pos++); @@ -104,8 +104,9 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) for (size_t n = 0 ; n < flag.arity; ++n) { if (pos == end) { if (flag.arity == ArityAny) break; - throw UsageError(format("flag '%1%' requires %2% argument(s)") - % name % flag.arity); + throw UsageError("flag '%1%' requires %2% argument(s)", + name, + flag.arity); } args.push_back(*pos++); } @@ -133,7 +134,7 @@ bool Args::processArgs(const Strings & args, bool finish) { if (expectedArgs.empty()) { if (!args.empty()) - throw UsageError(format("unexpected argument '%1%'") % args.front()); + throw UsageError("unexpected argument '%1%'", args.front()); return true; } diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 860b04adb..a117ddc72 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -481,7 +481,7 @@ ref makeCompressionSink(const std::string & method, Sink & next else if (method == "br") return make_ref(nextSink); else - throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); + throw UnknownCompressionMethod("unknown compression method '%s'", method); } ref compress(const std::string & method, const std::string & in, const bool parallel) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 08edffee0..d65215063 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -92,15 +92,15 @@ public: { fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); } + hintformat(const hintformat &hf) + : fmt(hf.fmt) + {} template hintformat& operator%(const T &value) { fmt % yellowify(value); return *this; } - hintformat(const hintformat &hf) - : fmt(hf.fmt) - {} std::string str() const diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 633c6bdf7..2261adb82 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -93,7 +93,9 @@ public: template BaseError(unsigned int status, const Args & ... args) - : err(hintfmt(args...)) + : err { .level = lvlError, + .hint = hintfmt(args...) + } , status(status) { } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 332c1c43a..bffcbcccf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -129,7 +129,7 @@ Path canonPath(const Path & path, bool resolveSymlinks) string s; if (path[0] != '/') - throw Error(format("not an absolute path: '%1%'") % path); + throw Error("not an absolute path: '%1%'", path); string::const_iterator i = path.begin(), end = path.end(); string temp; @@ -165,7 +165,7 @@ Path canonPath(const Path & path, bool resolveSymlinks) the symlink target might contain new symlinks). */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) - throw Error(format("infinite symlink recursion in path '%1%'") % path); + throw Error("infinite symlink recursion in path '%1%'", path); temp = absPath(readLink(s), dirOf(s)) + string(i, end); i = temp.begin(); /* restart */ @@ -226,7 +226,7 @@ struct stat lstat(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); return st; } @@ -238,7 +238,7 @@ bool pathExists(const Path & path) res = lstat(path.c_str(), &st); if (!res) return true; if (errno != ENOENT && errno != ENOTDIR) - throw SysError(format("getting status of %1%") % path); + throw SysError("getting status of %1%", path); return false; } @@ -274,7 +274,7 @@ DirEntries readDirectory(const Path & path) entries.reserve(64); AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -289,7 +289,7 @@ DirEntries readDirectory(const Path & path) #endif ); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return entries; } @@ -322,7 +322,7 @@ string readFile(const Path & path, bool drain) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); return drain ? drainFD(fd.get()) : readFile(fd.get()); } @@ -339,7 +339,7 @@ void writeFile(const Path & path, const string & s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); writeFull(fd.get(), s); } @@ -348,7 +348,7 @@ void writeFile(const Path & path, Source & source, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); std::vector buf(64 * 1024); @@ -396,7 +396,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) struct stat st; if (lstat(path.c_str(), &st) == -1) { if (errno == ENOENT) return; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) @@ -407,7 +407,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) - throw SysError(format("chmod '%1%'") % path); + throw SysError("chmod '%1%'", path); } for (auto & i : readDirectory(path)) @@ -416,7 +416,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) if (remove(path.c_str()) == -1) { if (errno == ENOENT) return; - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } } @@ -468,12 +468,12 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError(format("setting group of directory '%1%'") % tmpDir); + throw SysError("setting group of directory '%1%'", tmpDir); #endif return tmpDir; } if (errno != EEXIST) - throw SysError(format("creating directory '%1%'") % tmpDir); + throw SysError("creating directory '%1%'", tmpDir); } } @@ -555,15 +555,15 @@ Paths createDirs(const Path & path) if (lstat(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError(format("creating directory '%1%'") % path); + throw SysError("creating directory '%1%'", path); st = lstat(path); created.push_back(path); } if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError(format("statting symlink '%1%'") % path); + throw SysError("statting symlink '%1%'", path); - if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path); + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); return created; } @@ -572,7 +572,7 @@ Paths createDirs(const Path & path) void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) - throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + throw SysError("creating symlink from '%1%' to '%2%'", link, target); } @@ -589,7 +589,7 @@ void replaceSymlink(const Path & target, const Path & link) } if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); + throw SysError("renaming '%1%' to '%2%'", tmp, link); break; } @@ -694,7 +694,7 @@ AutoDelete::~AutoDelete() deletePath(path); else { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } } } catch (...) { @@ -760,7 +760,7 @@ void AutoCloseFD::close() if (fd != -1) { if (::close(fd) == -1) /* This should never happen. */ - throw SysError(format("closing file descriptor %1%") % fd); + throw SysError("closing file descriptor %1%", fd); } } @@ -920,7 +920,7 @@ void killUser(uid_t uid) #endif if (errno == ESRCH) break; /* no more processes */ if (errno != EINTR) - throw SysError(format("cannot kill processes for uid '%1%'") % uid); + throw SysError("cannot kill processes for uid '%1%'", uid); } _exit(0); @@ -928,7 +928,7 @@ void killUser(uid_t uid) int status = pid.wait(); if (status != 0) - throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status)); + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); /* !!! We should really do some check to make sure that there are no processes left running under `uid', but there is no portable @@ -1322,7 +1322,7 @@ void ignoreException() try { throw; } catch (std::exception & e) { - printError(format("error (ignored): %1%") % e.what()); + printError("error (ignored): %1%", e.what()); } } @@ -1440,7 +1440,7 @@ void callFailure(const std::function & failure, st try { failure(exc); } catch (std::exception & e) { - printError(format("uncaught exception: %s") % e.what()); + printError("uncaught exception: %s", e.what()); abort(); } } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index a2639579d..c403e3780 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -37,7 +37,8 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError(format("opening '%1%' for writing") % channelsList); + throw Error(""); + // throw SysError("opening '%1%' for writing", channelsList); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -46,9 +47,11 @@ static void writeChannels() static void addChannel(const string & url, const string & name) { if (!regex_search(url, std::regex("^(file|http|https)://"))) - throw Error(format("invalid channel URL '%1%'") % url); + throw Error(""); + // throw Error("invalid channel URL '%1%'", url); if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) - throw Error(format("invalid channel identifier '%1%'") % name); + throw Error(""); + // throw Error("invalid channel identifier '%1%'", name); readChannels(); channels[name] = url; writeChannels(); @@ -140,9 +143,11 @@ static void update(const StringSet & channelNames) if (S_ISLNK(st.st_mode)) // old-skool ~/.nix-defexpr if (unlink(nixDefExpr.c_str()) == -1) - throw SysError(format("unlinking %1%") % nixDefExpr); + throw Error(""); + // throw SysError("unlinking %1%", nixDefExpr); } else if (errno != ENOENT) { - throw SysError(format("getting status of %1%") % nixDefExpr); + throw Error(""); + // throw SysError("getting status of %1%", nixDefExpr); } createDirs(nixDefExpr); auto channelLink = nixDefExpr + "/channels"; @@ -194,10 +199,12 @@ static int _main(int argc, char ** argv) switch (cmd) { case cNone: - throw UsageError("no command specified"); + throw Error(""); + // throw UsageError("no command specified"); case cAdd: if (args.size() < 1 || args.size() > 2) - throw UsageError("'--add' requires one or two arguments"); + throw Error(""); + // throw UsageError("'--add' requires one or two arguments"); { auto url = args[0]; std::string name; @@ -213,12 +220,14 @@ static int _main(int argc, char ** argv) break; case cRemove: if (args.size() != 1) - throw UsageError("'--remove' requires one argument"); + throw Error(""); + // throw UsageError("'--remove' requires one argument"); removeChannel(args[0]); break; case cList: if (!args.empty()) - throw UsageError("'--list' expects no arguments"); + throw Error(""); + // throw UsageError("'--list' expects no arguments"); readChannels(); for (const auto & channel : channels) std::cout << channel.first << ' ' << channel.second << '\n'; @@ -228,7 +237,8 @@ static int _main(int argc, char ** argv) break; case cRollback: if (args.size() > 1) - throw UsageError("'--rollback' has at most one argument"); + throw Error(""); + // throw UsageError("'--rollback' has at most one argument"); Strings envArgs{"--profile", profile}; if (args.size() == 1) { envArgs.push_back("--switch-generation"); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index e68d1b1be..5f78ab464 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -36,7 +36,7 @@ using namespace nix::daemon; #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { - /* We ignore most parameters, we just have them for conformance with the linux syscall */ + // We ignore most parameters, we just have them for conformance with the linux syscall std::vector buf(8192); auto read_count = read(fd_in, buf.data(), buf.size()); if (read_count == -1) @@ -57,7 +57,7 @@ static void sigChldHandler(int sigNo) { // Ensure we don't modify errno of whatever we've interrupted auto saved_errno = errno; - /* Reap all dead children. */ + // Reap all dead children. while (waitpid(-1, 0, WNOHANG) > 0) ; errno = saved_errno; } @@ -106,7 +106,7 @@ struct PeerInfo }; -/* Get the identity of the caller, if possible. */ +// Get the identity of the caller, if possible. static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -154,13 +154,12 @@ static void daemonLoop(char * * argv) if (chdir("/") == -1) throw SysError("cannot change current directory"); - /* Get rid of children automatically; don't let them become - zombies. */ + // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); AutoCloseFD fdSocket; - /* Handle socket-based activation by systemd. */ + // Handle socket-based activation by systemd. auto listenFds = getEnv("LISTEN_FDS"); if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") @@ -169,17 +168,17 @@ static void daemonLoop(char * * argv) closeOnExec(fdSocket.get()); } - /* Otherwise, create and bind to a Unix domain socket. */ + // Otherwise, create and bind to a Unix domain socket. else { createDirs(dirOf(settings.nixDaemonSocketFile)); fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - /* Loop accepting connections. */ + // Loop accepting connections. while (1) { try { - /* Accept a connection. */ + // Accept a connection. struct sockaddr_un remoteAddr; socklen_t remoteAddrLen = sizeof(remoteAddr); @@ -209,13 +208,13 @@ static void daemonLoop(char * * argv) trusted = Trusted; if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) % (peer.pidKnown ? std::to_string(peer.pid) : "") % (peer.uidKnown ? user : "")); - /* Fork a child to handle the connection. */ + // Fork a child to handle the connection. ProcessOptions options; options.errorPrefix = "unexpected Nix daemon error: "; options.dieWithParent = false; @@ -224,20 +223,20 @@ static void daemonLoop(char * * argv) startProcess([&]() { fdSocket = -1; - /* Background the daemon. */ + // Background the daemon. if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); - /* Restore normal handling of SIGCHLD. */ + // Restore normal handling of SIGCHLD. setSigChldAction(false); - /* For debugging, stuff the pid into argv[1]. */ + // For debugging, stuff the pid into argv[1]. if (peer.pidKnown && argv[1]) { string processName = std::to_string(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } - /* Handle the connection. */ + // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); @@ -248,7 +247,7 @@ static void daemonLoop(char * * argv) } catch (Interrupted & e) { return; } catch (Error & e) { - printError(format("error processing connection: %1%") % e.msg()); + printError("error processing connection: %1%", e.msg()); } } } @@ -261,7 +260,7 @@ static int _main(int argc, char * * argv) parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") - ; /* ignored for backwards compatibility */ + ; // ignored for backwards compatibility else if (*arg == "--help") showManPage("nix-daemon"); else if (*arg == "--version") @@ -276,7 +275,7 @@ static int _main(int argc, char * * argv) if (stdio) { if (getStoreType() == tDaemon) { - /* Forward on this connection to the real daemon */ + // Forward on this connection to the real daemon auto socketPath = settings.nixDaemonSocketFile; auto s = socket(PF_UNIX, SOCK_STREAM, 0); if (s == -1) @@ -284,17 +283,17 @@ static int _main(int argc, char * * argv) auto socketDir = dirOf(socketPath); if (chdir(socketDir.c_str()) == -1) - throw SysError(format("changing to socket directory '%1%'") % socketDir); + throw SysError("changing to socket directory '%1%'", socketDir); auto socketName = std::string(baseNameOf(socketPath)); auto addr = sockaddr_un{}; addr.sun_family = AF_UNIX; if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket name %1% is too long") % socketName); + throw Error("socket name %1% is too long", socketName); strcpy(addr.sun_path, socketName.c_str()); if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at %1%") % socketPath); + throw SysError("cannot connect to daemon at %1%", socketPath); auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; while (true) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1a2bb42a3..dde8875f1 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -25,7 +25,6 @@ #include #include - using namespace nix; using std::cout; @@ -70,8 +69,7 @@ typedef void (* Operation) (Globals & globals, static string needArg(Strings::iterator & i, Strings & args, const string & arg) { - if (i == args.end()) throw UsageError( - format("'%1%' requires an argument") % arg); + if (i == args.end()) throw UsageError( "'%1%' requires an argument", arg); return *i++; } @@ -133,7 +131,7 @@ static void getAllExprs(EvalState & state, Value & vArg(*state.allocValue()); mkString(vArg, path2); if (v.attrs->size() == v.attrs->capacity()) - throw Error(format("too many Nix expressions in directory '%1%'") % path); + throw Error("too many Nix expressions in directory '%1%'", path); mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); } else if (S_ISDIR(st.st_mode)) @@ -144,11 +142,12 @@ static void getAllExprs(EvalState & state, } + static void loadSourceExpr(EvalState & state, const Path & path, Value & v) { struct stat st; if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting information about '%1%'") % path); + throw SysError("getting inon about '%1%'", path); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -221,7 +220,7 @@ static void checkSelectorUse(DrvNames & selectors) /* Check that all selectors have been used. */ for (auto & i : selectors) if (i.hits == 0 && i.fullName != "*") - throw Error(format("selector '%1%' matches no derivations") % i.fullName); + throw Error("selector '%1%' matches no derivations", i.fullName); } @@ -507,7 +506,7 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) globals.preserveInstalled = true; else if (arg == "--remove-all" || arg == "-r") globals.removeAll = true; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } installDerivations(globals, opArgs, globals.profile); @@ -618,7 +617,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--leq") upgradeType = utLeq; else if (arg == "--eq") upgradeType = utEq; else if (arg == "--always") upgradeType = utAlways; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } upgradeDerivations(globals, opArgs, upgradeType); @@ -637,7 +636,7 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv, static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() < 2) throw UsageError("not enough arguments to '--set-flag'"); @@ -680,7 +679,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { string arg = *i++; if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } DrvInfos elems; @@ -748,7 +747,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); uninstallDerivations(globals, opArgs, globals.profile); } @@ -911,7 +910,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--attr" || arg == "-A") attrPath = needArg(i, opFlags, arg); else - throw UsageError(format("unknown flag '%1%'") % arg); + throw UsageError("unknown flag '%1%'", arg); } if (printAttrPath && source != sAvailable) @@ -1177,9 +1176,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); Path profileLink = getHome() + "/.nix-profile"; @@ -1207,10 +1206,10 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) - throw Error(format("no generation older than the current (%1%) exists") - % curGen); + throw Error("no generation older than the current (%1%) exists", + curGen); else - throw Error(format("generation %1% does not exist") % dstGen); + throw Error("generation %1% does not exist", dstGen); } printInfo(format("switching from generation %1% to %2%") @@ -1225,13 +1224,13 @@ static void switchGeneration(Globals & globals, int dstGen) static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); int dstGen; if (!string2Int(opArgs.front(), dstGen)) - throw UsageError(format("expected a generation number")); + throw UsageError("expected a generation number"); switchGeneration(globals, dstGen); } @@ -1240,9 +1239,9 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); switchGeneration(globals, prevGen); } @@ -1251,9 +1250,9 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); PathLocks lock; lockProfile(lock, globals.profile); @@ -1278,7 +1277,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); @@ -1286,18 +1285,18 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { if(opArgs.front().size() < 2) - throw Error(format("invalid number of generations ‘%1%’") % opArgs.front()); + throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); int max; if (!string2Int(str_max, max) || max == 0) - throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front()); + throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { std::set gens; for (auto & i : opArgs) { unsigned int n; if (!string2Int(i, n)) - throw UsageError(format("invalid generation number '%1%'") % i); + throw UsageError("invalid generation number '%1%'", i); gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6c99d1181..bf353677a 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -66,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, /* What output do we want? */ string outputName = i.queryOutputName(); if (outputName == "") - throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath); + throw Error("derivation '%1%' lacks an 'outputName' attribute ", drvPath); if (gcRoot == "") printGCWarning(); @@ -166,7 +166,7 @@ static int _main(int argc, char * * argv) if (findFile) { for (auto & i : files) { Path p = state->findFile(i); - if (p == "") throw Error(format("unable to find '%1%'") % i); + if (p == "") throw Error("unable to find '%1%'", i); std::cout << p << std::endl; } return 0; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 2b9254659..d3d05cc50 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -37,11 +37,11 @@ string resolveMirrorUri(EvalState & state, string uri) auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); if (mirrorList == vMirrors.attrs->end()) - throw Error(format("unknown mirror name '%1%'") % mirrorName); + throw Error("unknown mirror name '%1%'", mirrorName); state.forceList(*mirrorList->value); if (mirrorList->value->listSize() < 1) - throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + throw Error("mirror URI '%1%' did not expand to anything", uri); string mirror = state.forceString(*mirrorList->value->listElems()[0]); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); @@ -73,7 +73,7 @@ static int _main(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--print-path") printPath = true; @@ -151,7 +151,7 @@ static int _main(int argc, char * * argv) if (name.empty()) name = baseNameOf(uri); if (name.empty()) - throw Error(format("cannot figure out file name for '%1%'") % uri); + throw Error("cannot figure out file name for '%1%'", uri); /* If an expected hash is given, the file may already exist in the store. */ @@ -206,7 +206,7 @@ static int _main(int argc, char * * argv) hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error(format("hash mismatch for '%1%'") % uri); + throw Error("hash mismatch for '%1%'", uri); /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fcc00175a..e4dd9bc96 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -124,7 +124,7 @@ static void opRealise(Strings opFlags, Strings opArgs) else if (i == "--repair") buildMode = bmRepair; else if (i == "--check") buildMode = bmCheck; else if (i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); std::vector paths; for (auto & i : opArgs) @@ -178,7 +178,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); @@ -198,10 +198,10 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) for (auto i : opFlags) if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) - throw UsageError(format("'--print-fixed-path' requires three arguments")); + throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); HashType hashAlgo = parseHashType(*i++); @@ -296,9 +296,9 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--use-output" || i == "-u") useOutput = true; else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (prev != qDefault && prev != query) - throw UsageError(format("query type '%1%' conflicts with earlier flag") % i); + throw UsageError("query type '%1%' conflicts with earlier flag", i); } if (query == qDefault) query = qOutputs; @@ -444,7 +444,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath)); /* Print each environment variable in the derivation in a format - that can be sourced by the shell. */ + * that can be sourced by the shell. */ for (auto & i : drv.env) cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); @@ -531,7 +531,7 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--reregister") reregister = true; else if (i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -545,7 +545,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) { auto path = store->followLinksToStorePath(i); @@ -576,7 +576,7 @@ static void opGC(Strings opFlags, Strings opArgs) long long maxFreed = getIntArg(*i, i, opFlags.end(), true); options.maxFreed = maxFreed >= 0 ? maxFreed : 0; } - else throw UsageError(format("bad sub-operation '%1%' in GC") % *i); + else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -612,7 +612,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); @@ -650,7 +650,7 @@ static void opRestore(Strings opFlags, Strings opArgs) static void opExport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); StorePathSet paths; @@ -666,7 +666,7 @@ static void opExport(Strings opFlags, Strings opArgs) static void opImport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -701,7 +701,7 @@ static void opVerify(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--check-contents") checkContents = true; else if (i == "--repair") repair = Repair; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { printError("warning: not all errors were fixed"); @@ -764,7 +764,7 @@ static void opServe(Strings opFlags, Strings opArgs) bool writeAllowed = false; for (auto & i : opFlags) if (i == "--write") writeAllowed = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -835,7 +835,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & p : willSubstitute) subs.emplace_back(p.clone()); store->buildPaths(subs); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + printError("warning: %1%", e.msg()); } } @@ -906,7 +906,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // informational only + auto drvPath = store->parseStorePath(readString(in)); // inonal ony BasicDerivation drv; readDerivation(in, *store, drv); @@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs) } default: - throw Error(format("unknown serve command %1%") % cmd); + throw Error("unknown serve command %1%", cmd); } out.flush(); @@ -973,7 +973,7 @@ static void opServe(Strings opFlags, Strings opArgs) static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) throw UsageError("three arguments expected"); auto i = opArgs.begin(); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 851f90abd..4e14f50ed 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -13,9 +13,9 @@ struct MixCat : virtual Args { auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); + throw Error("path '%1%' does not exist", path); if (st.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' is not a regular file") % path); + throw Error("path '%1%' is not a regular file", path); std::cout << accessor->readFile(path); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0cc523f50..128708339 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -126,7 +126,7 @@ static int compatNixHash(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--to-base16") op = opTo16; else if (*arg == "--to-base32") op = opTo32; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3ef1f2750..5d55bd4d5 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -65,7 +65,7 @@ struct MixLs : virtual Args, MixJSON auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); + throw Error("path '%1%' does not exist", path); doPath(st, path, st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 27727bd25..c770fd4a4 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -252,12 +252,12 @@ void NixRepl::mainLoop(const std::vector & files) // input without clearing the input so far. continue; } else { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } } catch (Error & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } catch (Interrupted & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } // We handled the current input fully, so we should clear it @@ -546,7 +546,7 @@ bool NixRepl::processLine(string line) return false; else if (command != "") - throw Error(format("unknown command '%1%'") % command); + throw Error("unknown command '%1%'", command); else { size_t p = line.find('='); diff --git a/src/nix/run.cc b/src/nix/run.cc index 8e30264c0..d790979a4 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -197,10 +197,10 @@ void chrootHelper(int argc, char * * argv) Finally freeCwd([&]() { free(cwd); }); if (chroot(tmpDir.c_str()) == -1) - throw SysError(format("chrooting into '%s'") % tmpDir); + throw SysError("chrooting into '%s'", tmpDir); if (chdir(cwd) == -1) - throw SysError(format("chdir to '%s' in chroot") % cwd); + throw SysError("chdir to '%s' in chroot", cwd); } else if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); From c0d940978a66841df638038e3ca85501bcee5734 Mon Sep 17 00:00:00 2001 From: Dustin DeWeese Date: Mon, 20 Apr 2020 17:32:50 -0700 Subject: [PATCH 144/350] Replace select() with poll() to allow waiting on more than FD_SETSIZE fds --- src/libstore/build.cc | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b4207e1b8..572634765 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -43,6 +42,7 @@ #include #include #include +#include #include #include @@ -4789,8 +4789,7 @@ void Worker::waitForInput() terminated. */ bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; + long timeout = 0; auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there @@ -4808,7 +4807,7 @@ void Worker::waitForInput() nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); + timeout = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); useTimeout = true; } @@ -4819,30 +4818,28 @@ void Worker::waitForInput() if (lastWokenUp == steady_time_point::min()) printError("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, + timeout = std::max(1L, (long) std::chrono::duration_cast( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); + vomit("sleeping %d seconds", timeout); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; + std::vector pollStatus; + std::map fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { - if (j >= FD_SETSIZE) - throw Error("reached FD_SETSIZE limit"); - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + fdToPollStatus[j] = pollStatus.size() - 1; } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (poll(pollStatus.data(), pollStatus.size(), + useTimeout ? timeout * 1000 : -1) == -1) { if (errno == EINTR) return; throw SysError("waiting for input"); } @@ -4863,7 +4860,7 @@ void Worker::waitForInput() set fds2(j->fds); std::vector buffer(4096); for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { + if (pollStatus.at(fdToPollStatus.at(k)).revents) { ssize_t rd = read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? From 2ea4d45449ea676e27cc678145ef39af7ac05ca8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 10:15:32 +0200 Subject: [PATCH 145/350] Path fetcher: Fix store path name (cherry picked from commit c7af247beacd418e6f2c4d33dffc35299101cd12) --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 7c7e20f4e..ba2cc192e 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -72,7 +72,7 @@ struct PathInput : Input if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) // FIXME: try to substitute storePath. - storePath = store->addToStore("name", path); + storePath = store->addToStore("source", path); return { From 7114f088fceb366e9ff0fb60e2970a11861d11fd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 14:54:16 +0200 Subject: [PATCH 146/350] Don't install error-demo --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f6b4a3101..e3057c36c 100644 --- a/Makefile +++ b/Makefile @@ -16,8 +16,7 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk \ - src/error-demo/local.mk + tests/plugins/local.mk -include Makefile.config From c9d0cf7e02d4b3af3c027d7d74d5caa3c8963d26 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 15:03:41 +0200 Subject: [PATCH 147/350] Don't include error.hh in util.hh to prevent header bloat --- src/libutil/util.hh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8e24ef968..d7ee62bcc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "logging.hh" #include "ansicolor.hh" -#include "error.hh" #include #include From 3bc9155dfc14434f5e9566fd15c79141899f6d07 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 22 Apr 2020 15:00:11 -0600 Subject: [PATCH 148/350] a few more 'format's rremoved --- src/libexpr/lexer.l | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index c34e5c383..85376a08f 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -127,14 +127,14 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError(format("invalid integer '%1%'") % yytext); + throw ParseError("invalid integer '%1%'", yytext); } return INT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError(format("invalid float '%1%'") % yytext); + throw ParseError("invalid float '%1%'", yytext); return FLOAT; } From 833501f6f11095ae226c13948eb3dc1de2a246ea Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 23 Apr 2020 15:55:34 -0600 Subject: [PATCH 149/350] 'what' string --- src/libutil/types.hh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 2261adb82..141bc6b3c 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -88,6 +88,13 @@ class BaseError : public std::exception protected: string prefix_; // used for location traces etc. ErrorInfo err; + string what_; + void initWhat() + { + std::ostringstream oss; + oss << err; + what_ = oss.str(); + } public: unsigned int status = 1; // exit status @@ -97,30 +104,27 @@ public: .hint = hintfmt(args...) } , status(status) - { - } + { initWhat(); } template BaseError(const Args & ... args) : err { .level = lvlError, .hint = hintfmt(args...) } - { - } + { initWhat(); } BaseError(ErrorInfo e) : err(e) - { - } + { initWhat(); } #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; - const char * what() const throw () { return err.description.c_str(); } + const char * what() const throw () { return what_.c_str(); } #else - const char * what() const noexcept { return err.description.c_str(); } + const char * what() const noexcept { return what_.c_str(); } #endif - const string & msg() const { return err.description; } + const string & msg() const { return what_; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); From d9632765a81604d270bf4693bddd53c22ee189d4 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 24 Apr 2020 12:44:23 -0600 Subject: [PATCH 150/350] add has_value check; remove obslete friend class --- src/libutil/error.cc | 2 +- src/libutil/fmt.hh | 5 ++--- src/libutil/logging.hh | 9 +++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 41595b0df..c6bca7135 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -162,7 +162,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) out << prefix << std::endl; // lines of code. - if (einfo.nixCode->errLineOfCode != "") { + if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") { printCodeLines(prefix, *einfo.nixCode); out << prefix << std::endl; } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index d65215063..d182adc3a 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -92,9 +92,11 @@ public: { fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); } + hintformat(const hintformat &hf) : fmt(hf.fmt) {} + template hintformat& operator%(const T &value) { @@ -102,14 +104,11 @@ public: return *this; } - std::string str() const { return fmt.str(); } - template - friend class AddHint; private: format fmt; }; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 8269e2e1a..89fd98419 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -62,11 +62,12 @@ public: virtual void logEI(const ErrorInfo &ei) = 0; - void logEI(Verbosity lvl, ErrorInfo ei) { - ei.level = lvl; - logEI(ei); + void logEI(Verbosity lvl, ErrorInfo ei) + { + ei.level = lvl; + logEI(ei); } - + virtual void warn(const std::string & msg); virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, From d8d4844b883dcea67cccb9bd75eecb14e60ae496 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 24 Apr 2020 14:57:51 -0600 Subject: [PATCH 151/350] all things error to error.hh --- src/libmain/shared.cc | 9 ++- src/libutil/error.cc | 18 +++++ src/libutil/types.hh | 151 ++---------------------------------------- src/libutil/util.cc | 18 ----- src/libutil/util.hh | 1 + 5 files changed, 30 insertions(+), 167 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f7ab56271..8551eb048 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -281,6 +281,8 @@ int handleExceptions(const string & programName, std::function fun) { ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + ErrorInfo::programName = programName; + string error = ANSI_RED "error:" ANSI_NORMAL " "; try { try { @@ -296,12 +298,15 @@ int handleExceptions(const string & programName, std::function fun) } catch (Exit & e) { return e.status; } catch (UsageError & e) { + // TODO: switch to logError + // logError(e.info()); printError( - format(error + "%1%\nTry '%2% --help' for more information.") + format("%1%\nTry '%2% --help' for more information.") % e.what() % programName); return 1; } catch (BaseError & e) { - printError(error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + // logError(e.info()); + printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); if (e.prefix() != "" && !settings.showTrace) printError("(use '--show-trace' to show detailed location information)"); return e.status; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c6bca7135..20b97ea3d 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -7,6 +7,24 @@ namespace nix { + +const std::string nativeSystem = SYSTEM; + + +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + + +std::string SysError::addErrno(const std::string & s) +{ + errNo = errno; + return s + ": " + strerror(errNo); +} + + std::optional ErrorInfo::programName = std::nullopt; std::ostream& operator<<(std::ostream &os, const hintformat &hf) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 141bc6b3c..f11256f61 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -5,20 +5,8 @@ #include #include -#include #include -#include -#include "fmt.hh" - -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif namespace nix { @@ -26,144 +14,13 @@ using std::list; using std::set; using std::vector; -typedef enum { - lvlError = 0, - lvlWarn, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - -struct ErrPos -{ - int line; - int column; - string file; - - template - ErrPos& operator=(const P &pos) - { - line = pos.line; - column = pos.column; - file = pos.file; - return *this; - } - - template - ErrPos(const P &p) - { - *this = p; - } -}; - -struct NixCode -{ - ErrPos errPos; - std::optional prevLineOfCode; - string errLineOfCode; - std::optional nextLineOfCode; -}; - -// ------------------------------------------------- -// ErrorInfo. -struct ErrorInfo -{ - Verbosity level; - string name; - string description; - std::optional hint; - std::optional nixCode; - - static std::optional programName; -}; - -std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); - -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ -class BaseError : public std::exception -{ -protected: - string prefix_; // used for location traces etc. - ErrorInfo err; - string what_; - void initWhat() - { - std::ostringstream oss; - oss << err; - what_ = oss.str(); - } -public: - unsigned int status = 1; // exit status - - template - BaseError(unsigned int status, const Args & ... args) - : err { .level = lvlError, - .hint = hintfmt(args...) - } - , status(status) - { initWhat(); } - - template - BaseError(const Args & ... args) - : err { .level = lvlError, - .hint = hintfmt(args...) - } - { initWhat(); } - - BaseError(ErrorInfo e) - : err(e) - { initWhat(); } - -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return what_.c_str(); } -#else - const char * what() const noexcept { return what_.c_str(); } -#endif - - const string & msg() const { return what_; } - const string & prefix() const { return prefix_; } - BaseError & addPrefix(const FormatOrString & fs); - - const ErrorInfo & info() const { return err; } -}; - -#define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - using superClass::superClass; \ - } - -MakeError(Error, BaseError); - -class SysError : public Error -{ -public: - int errNo; - - template - SysError(const Args & ... args) - : Error(args...) // TODO addErrNo for hintfmt - // : Error(addErrno(hintfmt(args...))) - { } - -private: - - std::string addErrno(const std::string & s); -}; - - -typedef list Strings; -typedef set StringSet; +typedef list Strings; +typedef set StringSet; typedef std::map StringMap; - /* Paths are just strings. */ -typedef string Path; + +typedef std::string Path; typedef list Paths; typedef set PathSet; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bffcbcccf..747a9e991 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -40,24 +40,6 @@ extern char * * environ; namespace nix { - -const std::string nativeSystem = SYSTEM; - - -BaseError & BaseError::addPrefix(const FormatOrString & fs) -{ - prefix_ = fs.s + prefix_; - return *this; -} - - -std::string SysError::addErrno(const std::string & s) -{ - errNo = errno; - return s + ": " + strerror(errNo); -} - - std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 636f3ba6c..978e14a30 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "error.hh" #include "logging.hh" #include "ansicolor.hh" From cdac083dc54b65f66f88caf193c617c1dbf3bbd6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 24 Apr 2020 21:40:13 -0600 Subject: [PATCH 152/350] don't print blank lines for blank description --- src/libutil/error.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 20b97ea3d..c800fe9a8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -88,7 +88,7 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) { int errwidth = 80; - string prefix = " "; + string prefix = ""; string levelString; switch (einfo.level) { @@ -176,8 +176,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) } // description - out << prefix << einfo.description << std::endl; - out << prefix << std::endl; + if (einfo.description != "") { + out << prefix << einfo.description << std::endl; + out << prefix << std::endl; + } // lines of code. if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") { From f59404e1a619934b9430b3a5bf352dbfffef2c31 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Sat, 25 Apr 2020 13:18:39 +0200 Subject: [PATCH 153/350] nix-env: refactor uninstallDerivations Reduces the number of store queries it performs. Also prints a warning if any of the selectors did not match any installed derivations. UX Caveats: - Will print a warning that nothing matched if a previous selector already removed the path - Will not do anything if no selectors were provided (no change from before). Fixes #3531 --- src/nix-env/nix-env.cc | 45 ++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1a2bb42a3..6c9cd9548 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -718,28 +718,39 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { string lockToken = optimisticLockProfile(profile); - DrvInfos installedElems = queryInstalled(*globals.state, profile); - DrvInfos newElems; + DrvInfos workingElems = queryInstalled(*globals.state, profile); - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - bool found = false; - for (auto & j : selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j)) - || DrvName(j).matches(drvName)) - { - printInfo("uninstalling '%s'", i.queryName()); - found = true; - break; - } - if (!found) newElems.push_back(i); + for (auto & selector : selectors) { + DrvInfos::iterator split = workingElems.begin(); + if (isPath(selector)) { + StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorStorePath, globals](auto &elem) { + return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath()); + } + ); + } else { + DrvName selectorName(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorName](auto &elem){ + DrvName elemName(elem.queryName()); + return !selectorName.matches(elemName); + } + ); + } + if (split == workingElems.end()) + warn("Selector '%s' matched no installed paths", selector); + for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { + printInfo("uninstalling '%s'", removedElem->queryName()); + } + workingElems.erase(split, workingElems.end()); } if (globals.dryRun) return; - if (createUserEnv(*globals.state, newElems, + if (createUserEnv(*globals.state, workingElems, profile, settings.envKeepDerivations, lockToken)) break; } } From d4fd7b543ee8dcea2a06c2f857b7e4dd72d34ccc Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sat, 25 Apr 2020 12:05:26 -0600 Subject: [PATCH 154/350] print dashes instead of empty name string --- src/libutil/error.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c800fe9a8..2c34325ac 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -148,14 +148,21 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) dashes.append("-"); // divider. - out << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL, - prefix, - levelString, - "---", - einfo.name, - dashes, - einfo.programName.value_or("")) - << std::endl; + if (einfo.name != "") + out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, + prefix, + levelString, + einfo.name, + dashes, + einfo.programName.value_or("")) + << std::endl; + else + out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, + prefix, + levelString, + dashes, + einfo.programName.value_or("")) + << std::endl; // filename. if (einfo.nixCode.has_value()) { From 1ff42722ce44f485c7dfabc6bb370494080f2755 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 26 Apr 2020 14:47:41 -0600 Subject: [PATCH 155/350] error.hh --- src/libutil/error.hh | 157 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 src/libutil/error.hh diff --git a/src/libutil/error.hh b/src/libutil/error.hh new file mode 100644 index 000000000..e33b1d3d6 --- /dev/null +++ b/src/libutil/error.hh @@ -0,0 +1,157 @@ +#pragma once + + +#include "ref.hh" + +#include +#include +#include +#include + +#include "fmt.hh" + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + +namespace nix { + +using std::list; +using std::vector; + +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +struct ErrPos +{ + int line; + int column; + string file; + + template + ErrPos& operator=(const P &pos) + { + line = pos.line; + column = pos.column; + file = pos.file; + return *this; + } + + template + ErrPos(const P &p) + { + *this = p; + } +}; + +struct NixCode +{ + ErrPos errPos; + std::optional prevLineOfCode; + string errLineOfCode; + std::optional nextLineOfCode; +}; + +// ------------------------------------------------- +// ErrorInfo. +struct ErrorInfo +{ + Verbosity level; + string name; + string description; + std::optional hint; + std::optional nixCode; + + static std::optional programName; +}; + +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + ErrorInfo err; + string what_; + void initWhat() + { + std::ostringstream oss; + oss << err; + what_ = oss.str(); + } +public: + unsigned int status = 1; // exit status + + template + BaseError(unsigned int status, const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(args...) + } + , status(status) + { initWhat(); } + + template + BaseError(const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(args...) + } + { initWhat(); } + + BaseError(ErrorInfo e) + : err(e) + { initWhat(); } + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() throw () { }; + const char * what() const throw () { return what_.c_str(); } +#else + const char * what() const noexcept { return what_.c_str(); } +#endif + + const string & msg() const { return what_; } + const string & prefix() const { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); + + const ErrorInfo & info() const { return err; } +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + using superClass::superClass; \ + } + +MakeError(Error, BaseError); + +class SysError : public Error +{ +public: + int errNo; + + template + SysError(const Args & ... args) + : Error(args...) // TODO addErrNo for hintfmt + // : Error(addErrno(hintfmt(args...))) + { } + +private: + + std::string addErrno(const std::string & s); +}; + +} From a3bc695e7dd60a0f1860942ba0343572b85e05e4 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 27 Apr 2020 11:12:54 -0600 Subject: [PATCH 156/350] Set GCROOT to store path to prevent garbage collection --- src/nix/shell.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 71e640667..bd07104cf 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -200,13 +200,15 @@ struct Common : InstallableCommand, MixProfile } } - BuildEnvironment getBuildEnvironment(ref store) + std::pair getBuildEnvironment(ref store) { auto shellOutPath = getShellOutPath(store); + auto strPath = store->printStorePath(shellOutPath); + updateProfile(shellOutPath); - return readEnvironment(store->printStorePath(shellOutPath)); + return {readEnvironment(strPath), strPath}; } }; @@ -253,7 +255,7 @@ struct CmdDevShell : Common, MixEnvironment void run(ref store) override { - auto buildEnvironment = getBuildEnvironment(store); + auto [buildEnvironment, gcroot] = getBuildEnvironment(store); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -276,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment auto shell = getEnv("SHELL").value_or("bash"); setEnviron(); + setenv("GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; @@ -307,7 +310,7 @@ struct CmdPrintDevEnv : Common void run(ref store) override { - auto buildEnvironment = getBuildEnvironment(store); + auto buildEnvironment = getBuildEnvironment(store).first; stopProgressBar(); From 9e95b95a5da59252d140fb61cb1177645935dcd6 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 27 Apr 2020 13:18:26 -0600 Subject: [PATCH 157/350] comment --- src/nix/shell.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index bd07104cf..fa8fe2e9b 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -278,6 +278,7 @@ struct CmdDevShell : Common, MixEnvironment auto shell = getEnv("SHELL").value_or("bash"); setEnviron(); + // prevent garbage collection until shell exits setenv("GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; From c05e20daa1abb3446e378331697938b78af2b3d7 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Mon, 27 Apr 2020 14:15:15 +0000 Subject: [PATCH 158/350] Fix long paths permanently breaking GC Suppose I have a path /nix/store/[hash]-[name]/a/a/a/a/a/[...]/a, long enough that everything after "/nix/store/" is longer than 4096 (MAX_PATH) bytes. Nix will happily allow such a path to be inserted into the store, because it doesn't look at all the nested structure. It just cares about the /nix/store/[hash]-[name] part. But, when the path is deleted, we encounter a problem. Nix will move the path to /nix/store/trash, but then when it's trying to recursively delete the trash directory, it will at some point try to unlink /nix/store/trash/[hash]-[name]/a/a/a/a/a/[...]/a. This will fail, because the path is too long. After this has failed, any store deletion operation will never work again, because Nix needs to delete the trash directory before recreating it to move new things to it. (I assume this is because otherwise a path being deleted could already exist in the trash, and then moving it would fail.) This means that if I can trick somebody into just fetching a tarball containing a path of the right length, they won't be able to delete store paths or garbage collect ever again, until the offending path is manually removed from /nix/store/trash. (And even fixing this manually is quite difficult if you don't understand the issue, because the absolute path that Nix says it failed to remove is also too long for rm(1).) This patch fixes the issue by making Nix's recursive delete operation use unlinkat(2). This function takes a relative path and a directory file descriptor. We ensure that the relative path is always just the name of the directory entry, and therefore its length will never exceed 255 bytes. This means that it will never even come close to AX_PATH, and Nix will therefore be able to handle removing arbitrarily deep directory hierachies. Since the directory file descriptor is used for recursion after being used in readDirectory, I made a variant of readDirectory that takes an already open directory stream, to avoid the directory being opened multiple times. As we have seen from this issue, the less we have to interact with paths, the better, and so it's good to reuse file descriptors where possible. I left _deletePath as succeeding even if the parent directory doesn't exist, even though that feels wrong to me, because without that early return, the linux-sandbox test failed. Reported-by: Alyssa Ross Thanks-to: Puck Meerburg Tested-by: Puck Meerburg Reviewed-by: Puck Meerburg --- src/libutil/util.cc | 54 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 332c1c43a..615a7656c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -268,16 +268,13 @@ bool isLink(const Path & path) } -DirEntries readDirectory(const Path & path) +DirEntries readDirectory(DIR *dir, const Path & path) { DirEntries entries; entries.reserve(64); - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + while (errno = 0, dirent = readdir(dir)) { /* sic */ checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -294,6 +291,14 @@ DirEntries readDirectory(const Path & path) return entries; } +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); + + return readDirectory(dir.get(), path); +} + unsigned char getFileType(const Path & path) { @@ -389,12 +394,14 @@ void writeLine(int fd, string s) } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); + string name(baseNameOf(path)); + struct stat st; - if (lstat(path.c_str(), &st) == -1) { + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; throw SysError(format("getting status of '%1%'") % path); } @@ -406,20 +413,45 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) /* Make the directory accessible. */ const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) throw SysError(format("chmod '%1%'") % path); } - for (auto & i : readDirectory(path)) - _deletePath(path + "/" + i.name, bytesFreed); + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (!fd) + throw SysError(format("opening directory '%1%'") % path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError(format("opening directory '%1%'") % path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) { + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; throw SysError(format("cannot unlink '%1%'") % path); } } +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + // This really shouldn't fail silently, but it's left this way + // for backwards compatibility. + if (errno == ENOENT) return; + + throw SysError(format("opening directory '%1%'") % path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + void deletePath(const Path & path) { From e51a757720ade58e48128730134400cf12f54851 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 27 Apr 2020 15:15:08 -0600 Subject: [PATCH 159/350] astyle format --- src/libutil/error.hh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index e33b1d3d6..b715c3e45 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -24,7 +24,8 @@ namespace nix { using std::list; using std::vector; -typedef enum { +typedef enum +{ lvlError = 0, lvlWarn, lvlInfo, @@ -52,7 +53,7 @@ struct ErrPos template ErrPos(const P &p) { - *this = p; + *this = p; } }; @@ -87,7 +88,7 @@ protected: string prefix_; // used for location traces etc. ErrorInfo err; string what_; - void initWhat() + void initWhat() { std::ostringstream oss; oss << err; @@ -106,7 +107,7 @@ public: template BaseError(const Args & ... args) - : err { .level = lvlError, + : err { .level = lvlError, .hint = hintfmt(args...) } { initWhat(); } @@ -146,7 +147,7 @@ public: template SysError(const Args & ... args) : Error(args...) // TODO addErrNo for hintfmt - // : Error(addErrno(hintfmt(args...))) + // : Error(addErrno(hintfmt(args...))) { } private: From 52a3ca823dd8aca8da32a198be2b1238cb321e6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 28 Apr 2020 17:56:01 +0200 Subject: [PATCH 160/350] Tweak warning message --- src/nix-env/nix-env.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 6c9cd9548..d62febaff 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -741,7 +741,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, ); } if (split == workingElems.end()) - warn("Selector '%s' matched no installed paths", selector); + warn("selector '%s' matched no installed derivations", selector); for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { printInfo("uninstalling '%s'", removedElem->queryName()); } From 6d40fe573c859725cac6eb2c884ecf84965c8853 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Tue, 28 Apr 2020 11:18:54 -0600 Subject: [PATCH 161/350] rename to NIX_GCROOT --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index fa8fe2e9b..fbfe24bbe 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -279,7 +279,7 @@ struct CmdDevShell : Common, MixEnvironment setEnviron(); // prevent garbage collection until shell exits - setenv("GCROOT", gcroot.data(), 1); + setenv("NIX_GCROOT", gcroot.data(), 1); auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; From 22e6490311015e076a09d5608834bd0dec1d7020 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 28 Apr 2020 21:06:08 -0600 Subject: [PATCH 162/350] Error classname as name --- src/error-demo/error-demo.cc | 16 +++++++++--- src/libstore/sqlite.cc | 2 +- src/libstore/sqlite.hh | 2 +- src/libutil/error.cc | 29 +++++++++++----------- src/libutil/error.hh | 47 +++++++++++++++++++++--------------- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 14027278d..98018ca0e 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -4,10 +4,12 @@ #include #include +using namespace nix; + +MakeError(DemoError, Error); + int main() { - using namespace nix; - makeDefaultLogger(); verbosity = lvlVomit; @@ -15,6 +17,12 @@ int main() // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); + try { + throw DemoError("demo error was thrown"); + } catch (Error &e) { + logger->logEI(e.info()); + } + // For completeness sake, info through vomit levels. // But this is maybe a heavy format for those. logger->logEI( @@ -79,7 +87,7 @@ int main() .prevLineOfCode = std::nullopt, .errLineOfCode = "this is the problem line of code", .nextLineOfCode = std::nullopt - }}); + }}); // Error with previous and next lines of code. logError( @@ -93,7 +101,7 @@ int main() .prevLineOfCode = std::optional("previous line of code"), .errLineOfCode = "this is the problem line of code", .nextLineOfCode = std::optional("next line of code"), - }}); + }}); return 0; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 3407e5826..a72cd5d88 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -196,7 +196,7 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(SQLiteBusy & e) { static std::atomic lastWarned{0}; diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index fd04c9b07..ce27033c9 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -99,7 +99,7 @@ MakeError(SQLiteBusy, SQLiteError); [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(SQLiteBusy & e); /* Convenience function for retrying a SQLite transaction when the database is busy. */ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 2c34325ac..8f0e0aa1a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -4,8 +4,7 @@ #include #include "serialise.hh" -namespace nix -{ +namespace nix { const std::string nativeSystem = SYSTEM; @@ -149,20 +148,20 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) // divider. if (einfo.name != "") - out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, - prefix, - levelString, - einfo.name, - dashes, - einfo.programName.value_or("")) - << std::endl; + out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, + prefix, + levelString, + einfo.name, + dashes, + einfo.programName.value_or("")) + << std::endl; else - out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, - prefix, - levelString, - dashes, - einfo.programName.value_or("")) - << std::endl; + out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, + prefix, + levelString, + dashes, + einfo.programName.value_or("")) + << std::endl; // filename. if (einfo.nixCode.has_value()) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index b715c3e45..03e43241f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -24,8 +24,7 @@ namespace nix { using std::list; using std::vector; -typedef enum -{ +typedef enum { lvlError = 0, lvlWarn, lvlInfo, @@ -35,8 +34,7 @@ typedef enum lvlVomit } Verbosity; -struct ErrPos -{ +struct ErrPos { int line; int column; string file; @@ -57,8 +55,7 @@ struct ErrPos } }; -struct NixCode -{ +struct NixCode { ErrPos errPos; std::optional prevLineOfCode; string errLineOfCode; @@ -67,8 +64,7 @@ struct NixCode // ------------------------------------------------- // ErrorInfo. -struct ErrorInfo -{ +struct ErrorInfo { Verbosity level; string name; string description; @@ -87,12 +83,20 @@ class BaseError : public std::exception protected: string prefix_; // used for location traces etc. ErrorInfo err; - string what_; - void initWhat() + std::optional what_; + const string& calcWhat() { - std::ostringstream oss; - oss << err; - what_ = oss.str(); + if (what_.has_value()) + return *what_; + else { + err.name = sname(); + + std::ostringstream oss; + oss << err; + what_ = oss.str(); + + return *what_; + } } public: unsigned int status = 1; // exit status @@ -103,31 +107,33 @@ public: .hint = hintfmt(args...) } , status(status) - { initWhat(); } + { } template BaseError(const Args & ... args) : err { .level = lvlError, .hint = hintfmt(args...) } - { initWhat(); } + { } BaseError(ErrorInfo e) : err(e) - { initWhat(); } + { } + + virtual const char* sname() const { return "BaseError"; } #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; - const char * what() const throw () { return what_.c_str(); } + const char * what() throw () { return calcWhat().c_str(); } #else - const char * what() const noexcept { return what_.c_str(); } + const char * what() noexcept { return calcWhat().c_str(); } #endif - const string & msg() const { return what_; } + const string & msg() { return calcWhat(); } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); - const ErrorInfo & info() const { return err; } + const ErrorInfo & info() { calcWhat(); return err; } }; #define MakeError(newClass, superClass) \ @@ -135,6 +141,7 @@ public: { \ public: \ using superClass::superClass; \ + virtual const char* sname() const override { return #newClass; } \ } MakeError(Error, BaseError); From 5a34a473dd14200ffeff496f0e8e4518f7992765 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 13:10:03 +0200 Subject: [PATCH 163/350] builtins.readFile: do not truncate content This closes #3026 by allowing `builtins.readFile` to read a file with a wrongly reported file size, for example, files in `/proc` may report a file size of 0. Reading file in `/proc` is not a good enough motivation, however I do think it just makes nix more robust by allowing more file to be read. Especially, I do considerer the previous behavior to be dangerous because nix was previously reading truncated files. Examples of file system which incorrectly report file size may be network file system or dynamic file system (for performance reason, a dynamic file system such as FUSE may generate the content of the file on demand). ``` nix-repl> builtins.readFile "/proc/version" "" ``` With this commit: ``` nix-repl> builtins.readFile "/proc/version" "Linux version 5.6.7 (nixbld@localhost) (gcc version 9.3.0 (GCC)) #1-NixOS SMP Thu Apr 23 08:38:27 UTC 2020\n" ``` Here is a summary of the behavior changes: - If the reported size is smaller, previous implementation was silently returning a truncated file content. The new implementation is returning the correct file content. - If a file had a bigger reported file size, previous implementation was failing with an exception, but the new implementation is returning the correct file content. This change of behavior is coherent with this pull request. Open questions - The behavior is unchanged for correctly reported file size, however performances may vary because it uses the more complex sink interface. Considering that sink is used a lot, I don't think this impacts the performance a lot. - `builtins.readFile` on an infinite file, such as `/dev/random` may fill the memory. - it does not support adding file to store, such as `${/proc/version}`. --- src/libutil/util.cc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 615a7656c..d001bc3b7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -312,14 +312,7 @@ unsigned char getFileType(const Path & path) string readFile(int fd) { - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - std::vector buf(st.st_size); - readFull(fd, buf.data(), st.st_size); - - return string((char *) buf.data(), st.st_size); + return drainFD(fd, true); } From ca93b26db6d2d73e702ea8ecdd0a98f17ace2c7b Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Wed, 29 Apr 2020 14:39:37 +0200 Subject: [PATCH 164/350] Only call grantpt on MacOS systems The commit 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 adds a `grantpt` call on the builder pseudo terminal fd. This call is actually only required for MacOS, but it however requires a RW access to /dev/pts which is only RO bindmounted in the Bazel Linux sandbox. So, Nix can not be actually run in the Bazel Linux sandbox for unneeded reasons. --- src/libstore/build.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 572634765..147093fae 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2250,10 +2250,13 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); - } else { + } +#if __APPLE__ + else { if (grantpt(builderOut.readSide.get())) throw SysError("granting access to pseudoterminal slave"); } +#endif #if 0 // Mount the pt in the sandbox so that the "tty" command works. From e2f61263ebf41958f4f3d6f6135377024e33457c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 29 Apr 2020 10:14:32 -0600 Subject: [PATCH 165/350] uncrustify formatting --- .uncrustify.cfg | 2989 ++++++++++++++++++++++++++++++++++ src/error-demo/error-demo.cc | 26 +- src/libexpr/nixexpr.hh | 3 +- src/libutil/error.cc | 48 +- src/libutil/logging.cc | 42 +- src/libutil/logging.hh | 4 +- 6 files changed, 3050 insertions(+), 62 deletions(-) create mode 100644 .uncrustify.cfg diff --git a/.uncrustify.cfg b/.uncrustify.cfg new file mode 100644 index 000000000..23305adcf --- /dev/null +++ b/.uncrustify.cfg @@ -0,0 +1,2989 @@ +# Uncrustify-0.70.1_f + +# +# General options +# + +# The type of line endings. +# +# Default: auto +newlines = auto # lf/crlf/cr/auto + +# The original size of tabs in the input. +# +# Default: 8 +input_tab_size = 4 # unsigned number + +# The size of tabs in the output (only used if align_with_tabs=true). +# +# Default: 8 +output_tab_size = 4 # unsigned number + +# The ASCII value of the string escape char, usually 92 (\) or (Pawn) 94 (^). +# +# Default: 92 +string_escape_char = 92 # unsigned number + +# Alternate string escape char (usually only used for Pawn). +# Only works right before the quote char. +string_escape_char2 = 0 # unsigned number + +# Replace tab characters found in string literals with the escape sequence \t +# instead. +string_replace_tab_chars = false # true/false + +# Allow interpreting '>=' and '>>=' as part of a template in code like +# 'void f(list>=val);'. If true, 'assert(x<0 && y>=3)' will be broken. +# Improvements to template detection may make this option obsolete. +tok_split_gte = false # true/false + +# Specify the marker used in comments to disable processing of part of the +# file. +# +# Default: *INDENT-OFF* +disable_processing_cmt = " *INDENT-OFF*" # string + +# Specify the marker used in comments to (re)enable processing in a file. +# +# Default: *INDENT-ON* +enable_processing_cmt = " *INDENT-ON*" # string + +# Enable parsing of digraphs. +enable_digraphs = false # true/false + +# Add or remove the UTF-8 BOM (recommend 'remove'). +utf8_bom = ignore # ignore/add/remove/force + +# If the file contains bytes with values between 128 and 255, but is not +# UTF-8, then output as UTF-8. +utf8_byte = false # true/false + +# Force the output encoding to UTF-8. +utf8_force = false # true/false + +# Add or remove space between 'do' and '{'. +sp_do_brace_open = ignore # ignore/add/remove/force + +# Add or remove space between '}' and 'while'. +sp_brace_close_while = ignore # ignore/add/remove/force + +# Add or remove space between 'while' and '('. +sp_while_paren_open = ignore # ignore/add/remove/force + +# +# Spacing options +# + +# Add or remove space around non-assignment symbolic operators ('+', '/', '%', +# '<<', and so forth). +sp_arith = ignore # ignore/add/remove/force + +# Add or remove space around arithmetic operators '+' and '-'. +# +# Overrides sp_arith. +sp_arith_additive = ignore # ignore/add/remove/force + +# Add or remove space around assignment operator '=', '+=', etc. +sp_assign = ignore # ignore/add/remove/force + +# Add or remove space around '=' in C++11 lambda capture specifications. +# +# Overrides sp_assign. +sp_cpp_lambda_assign = ignore # ignore/add/remove/force + +# Add or remove space after the capture specification of a C++11 lambda when +# an argument list is present, as in '[] (int x){ ... }'. +sp_cpp_lambda_square_paren = ignore # ignore/add/remove/force + +# Add or remove space after the capture specification of a C++11 lambda with +# no argument list is present, as in '[] { ... }'. +sp_cpp_lambda_square_brace = ignore # ignore/add/remove/force + +# Add or remove space after the argument list of a C++11 lambda, as in +# '[](int x) { ... }'. +sp_cpp_lambda_paren_brace = ignore # ignore/add/remove/force + +# Add or remove space between a lambda body and its call operator of an +# immediately invoked lambda, as in '[]( ... ){ ... } ( ... )'. +sp_cpp_lambda_fparen = ignore # ignore/add/remove/force + +# Add or remove space around assignment operator '=' in a prototype. +# +# If set to ignore, use sp_assign. +sp_assign_default = ignore # ignore/add/remove/force + +# Add or remove space before assignment operator '=', '+=', etc. +# +# Overrides sp_assign. +sp_before_assign = ignore # ignore/add/remove/force + +# Add or remove space after assignment operator '=', '+=', etc. +# +# Overrides sp_assign. +sp_after_assign = ignore # ignore/add/remove/force + +# Add or remove space in 'NS_ENUM ('. +sp_enum_paren = ignore # ignore/add/remove/force + +# Add or remove space around assignment '=' in enum. +sp_enum_assign = ignore # ignore/add/remove/force + +# Add or remove space before assignment '=' in enum. +# +# Overrides sp_enum_assign. +sp_enum_before_assign = ignore # ignore/add/remove/force + +# Add or remove space after assignment '=' in enum. +# +# Overrides sp_enum_assign. +sp_enum_after_assign = ignore # ignore/add/remove/force + +# Add or remove space around assignment ':' in enum. +sp_enum_colon = ignore # ignore/add/remove/force + +# Add or remove space around preprocessor '##' concatenation operator. +# +# Default: add +sp_pp_concat = add # ignore/add/remove/force + +# Add or remove space after preprocessor '#' stringify operator. +# Also affects the '#@' charizing operator. +sp_pp_stringify = ignore # ignore/add/remove/force + +# Add or remove space before preprocessor '#' stringify operator +# as in '#define x(y) L#y'. +sp_before_pp_stringify = ignore # ignore/add/remove/force + +# Add or remove space around boolean operators '&&' and '||'. +sp_bool = ignore # ignore/add/remove/force + +# Add or remove space around compare operator '<', '>', '==', etc. +sp_compare = ignore # ignore/add/remove/force + +# Add or remove space inside '(' and ')'. +sp_inside_paren = ignore # ignore/add/remove/force + +# Add or remove space between nested parentheses, i.e. '((' vs. ') )'. +sp_paren_paren = ignore # ignore/add/remove/force + +# Add or remove space between back-to-back parentheses, i.e. ')(' vs. ') ('. +sp_cparen_oparen = ignore # ignore/add/remove/force + +# Whether to balance spaces inside nested parentheses. +sp_balance_nested_parens = false # true/false + +# Add or remove space between ')' and '{'. +sp_paren_brace = ignore # ignore/add/remove/force + +# Add or remove space between nested braces, i.e. '{{' vs '{ {'. +sp_brace_brace = ignore # ignore/add/remove/force + +# Add or remove space before pointer star '*'. +sp_before_ptr_star = ignore # ignore/add/remove/force + +# Add or remove space before pointer star '*' that isn't followed by a +# variable name. If set to ignore, sp_before_ptr_star is used instead. +sp_before_unnamed_ptr_star = ignore # ignore/add/remove/force + +# Add or remove space between pointer stars '*'. +sp_between_ptr_star = ignore # ignore/add/remove/force + +# Add or remove space after pointer star '*', if followed by a word. +# +# Overrides sp_type_func. +sp_after_ptr_star = ignore # ignore/add/remove/force + +# Add or remove space after pointer caret '^', if followed by a word. +sp_after_ptr_block_caret = ignore # ignore/add/remove/force + +# Add or remove space after pointer star '*', if followed by a qualifier. +sp_after_ptr_star_qualifier = ignore # ignore/add/remove/force + +# Add or remove space after a pointer star '*', if followed by a function +# prototype or function definition. +# +# Overrides sp_after_ptr_star and sp_type_func. +sp_after_ptr_star_func = ignore # ignore/add/remove/force + +# Add or remove space after a pointer star '*', if followed by an open +# parenthesis, as in 'void* (*)(). +sp_ptr_star_paren = ignore # ignore/add/remove/force + +# Add or remove space before a pointer star '*', if followed by a function +# prototype or function definition. +sp_before_ptr_star_func = ignore # ignore/add/remove/force + +# Add or remove space before a reference sign '&'. +sp_before_byref = ignore # ignore/add/remove/force + +# Add or remove space before a reference sign '&' that isn't followed by a +# variable name. If set to ignore, sp_before_byref is used instead. +sp_before_unnamed_byref = ignore # ignore/add/remove/force + +# Add or remove space after reference sign '&', if followed by a word. +# +# Overrides sp_type_func. +sp_after_byref = ignore # ignore/add/remove/force + +# Add or remove space after a reference sign '&', if followed by a function +# prototype or function definition. +# +# Overrides sp_after_byref and sp_type_func. +sp_after_byref_func = ignore # ignore/add/remove/force + +# Add or remove space before a reference sign '&', if followed by a function +# prototype or function definition. +sp_before_byref_func = ignore # ignore/add/remove/force + +# Add or remove space between type and word. +# +# Default: force +sp_after_type = force # ignore/add/remove/force + +# Add or remove space between 'decltype(...)' and word. +sp_after_decltype = ignore # ignore/add/remove/force + +# (D) Add or remove space before the parenthesis in the D constructs +# 'template Foo(' and 'class Foo('. +sp_before_template_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'template' and '<'. +# If set to ignore, sp_before_angle is used. +sp_template_angle = ignore # ignore/add/remove/force + +# Add or remove space before '<'. +sp_before_angle = ignore # ignore/add/remove/force + +# Add or remove space inside '<' and '>'. +sp_inside_angle = ignore # ignore/add/remove/force + +# Add or remove space inside '<>'. +sp_inside_angle_empty = ignore # ignore/add/remove/force + +# Add or remove space between '>' and ':'. +sp_angle_colon = ignore # ignore/add/remove/force + +# Add or remove space after '>'. +sp_after_angle = ignore # ignore/add/remove/force + +# Add or remove space between '>' and '(' as found in 'new List(foo);'. +sp_angle_paren = ignore # ignore/add/remove/force + +# Add or remove space between '>' and '()' as found in 'new List();'. +sp_angle_paren_empty = ignore # ignore/add/remove/force + +# Add or remove space between '>' and a word as in 'List m;' or +# 'template static ...'. +sp_angle_word = ignore # ignore/add/remove/force + +# Add or remove space between '>' and '>' in '>>' (template stuff). +# +# Default: add +sp_angle_shift = add # ignore/add/remove/force + +# (C++11) Permit removal of the space between '>>' in 'foo >'. Note +# that sp_angle_shift cannot remove the space without this option. +sp_permit_cpp11_shift = false # true/false + +# Add or remove space before '(' of control statements ('if', 'for', 'switch', +# 'while', etc.). +sp_before_sparen = ignore # ignore/add/remove/force + +# Add or remove space inside '(' and ')' of control statements. +sp_inside_sparen = ignore # ignore/add/remove/force + +# Add or remove space after '(' of control statements. +# +# Overrides sp_inside_sparen. +sp_inside_sparen_open = ignore # ignore/add/remove/force + +# Add or remove space before ')' of control statements. +# +# Overrides sp_inside_sparen. +sp_inside_sparen_close = ignore # ignore/add/remove/force + +# Add or remove space after ')' of control statements. +sp_after_sparen = ignore # ignore/add/remove/force + +# Add or remove space between ')' and '{' of of control statements. +sp_sparen_brace = ignore # ignore/add/remove/force + +# (D) Add or remove space between 'invariant' and '('. +sp_invariant_paren = ignore # ignore/add/remove/force + +# (D) Add or remove space after the ')' in 'invariant (C) c'. +sp_after_invariant_paren = ignore # ignore/add/remove/force + +# Add or remove space before empty statement ';' on 'if', 'for' and 'while'. +sp_special_semi = ignore # ignore/add/remove/force + +# Add or remove space before ';'. +# +# Default: remove +sp_before_semi = remove # ignore/add/remove/force + +# Add or remove space before ';' in non-empty 'for' statements. +sp_before_semi_for = ignore # ignore/add/remove/force + +# Add or remove space before a semicolon of an empty part of a for statement. +sp_before_semi_for_empty = ignore # ignore/add/remove/force + +# Add or remove space after ';', except when followed by a comment. +# +# Default: add +sp_after_semi = add # ignore/add/remove/force + +# Add or remove space after ';' in non-empty 'for' statements. +# +# Default: force +sp_after_semi_for = force # ignore/add/remove/force + +# Add or remove space after the final semicolon of an empty part of a for +# statement, as in 'for ( ; ; )'. +sp_after_semi_for_empty = ignore # ignore/add/remove/force + +# Add or remove space before '[' (except '[]'). +sp_before_square = ignore # ignore/add/remove/force + +# Add or remove space before '[' for a variable definition. +# +# Default: remove +sp_before_vardef_square = remove # ignore/add/remove/force + +# Add or remove space before '[' for asm block. +sp_before_square_asm_block = ignore # ignore/add/remove/force + +# Add or remove space before '[]'. +sp_before_squares = ignore # ignore/add/remove/force + +# Add or remove space before C++17 structured bindings. +sp_cpp_before_struct_binding = ignore # ignore/add/remove/force + +# Add or remove space inside a non-empty '[' and ']'. +sp_inside_square = ignore # ignore/add/remove/force + +# (OC) Add or remove space inside a non-empty Objective-C boxed array '@[' and +# ']'. If set to ignore, sp_inside_square is used. +sp_inside_square_oc_array = ignore # ignore/add/remove/force + +# Add or remove space after ',', i.e. 'a,b' vs. 'a, b'. +sp_after_comma = ignore # ignore/add/remove/force + +# Add or remove space before ','. +# +# Default: remove +sp_before_comma = remove # ignore/add/remove/force + +# (C#) Add or remove space between ',' and ']' in multidimensional array type +# like 'int[,,]'. +sp_after_mdatype_commas = ignore # ignore/add/remove/force + +# (C#) Add or remove space between '[' and ',' in multidimensional array type +# like 'int[,,]'. +sp_before_mdatype_commas = ignore # ignore/add/remove/force + +# (C#) Add or remove space between ',' in multidimensional array type +# like 'int[,,]'. +sp_between_mdatype_commas = ignore # ignore/add/remove/force + +# Add or remove space between an open parenthesis and comma, +# i.e. '(,' vs. '( ,'. +# +# Default: force +sp_paren_comma = force # ignore/add/remove/force + +# Add or remove space before the variadic '...' when preceded by a +# non-punctuator. +sp_before_ellipsis = remove # ignore/add/remove/force + +# Add or remove space between a type and '...'. +sp_type_ellipsis = ignore # ignore/add/remove/force + +# (D) Add or remove space between a type and '?'. +sp_type_question = ignore # ignore/add/remove/force + +# Add or remove space between ')' and '...'. +sp_paren_ellipsis = ignore # ignore/add/remove/force + +# Add or remove space between ')' and a qualifier such as 'const'. +sp_paren_qualifier = ignore # ignore/add/remove/force + +# Add or remove space between ')' and 'noexcept'. +sp_paren_noexcept = ignore # ignore/add/remove/force + +# Add or remove space after class ':'. +sp_after_class_colon = ignore # ignore/add/remove/force + +# Add or remove space before class ':'. +sp_before_class_colon = ignore # ignore/add/remove/force + +# Add or remove space after class constructor ':'. +sp_after_constr_colon = ignore # ignore/add/remove/force + +# Add or remove space before class constructor ':'. +sp_before_constr_colon = ignore # ignore/add/remove/force + +# Add or remove space before case ':'. +# +# Default: remove +sp_before_case_colon = remove # ignore/add/remove/force + +# Add or remove space between 'operator' and operator sign. +sp_after_operator = ignore # ignore/add/remove/force + +# Add or remove space between the operator symbol and the open parenthesis, as +# in 'operator ++('. +sp_after_operator_sym = ignore # ignore/add/remove/force + +# Overrides sp_after_operator_sym when the operator has no arguments, as in +# 'operator *()'. +sp_after_operator_sym_empty = ignore # ignore/add/remove/force + +# Add or remove space after C/D cast, i.e. 'cast(int)a' vs. 'cast(int) a' or +# '(int)a' vs. '(int) a'. +sp_after_cast = ignore # ignore/add/remove/force + +# Add or remove spaces inside cast parentheses. +sp_inside_paren_cast = ignore # ignore/add/remove/force + +# Add or remove space between the type and open parenthesis in a C++ cast, +# i.e. 'int(exp)' vs. 'int (exp)'. +sp_cpp_cast_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'sizeof' and '('. +sp_sizeof_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'sizeof' and '...'. +sp_sizeof_ellipsis = ignore # ignore/add/remove/force + +# Add or remove space between 'sizeof...' and '('. +sp_sizeof_ellipsis_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'decltype' and '('. +sp_decltype_paren = ignore # ignore/add/remove/force + +# (Pawn) Add or remove space after the tag keyword. +sp_after_tag = ignore # ignore/add/remove/force + +# Add or remove space inside enum '{' and '}'. +sp_inside_braces_enum = ignore # ignore/add/remove/force + +# Add or remove space inside struct/union '{' and '}'. +sp_inside_braces_struct = ignore # ignore/add/remove/force + +# (OC) Add or remove space inside Objective-C boxed dictionary '{' and '}' +sp_inside_braces_oc_dict = ignore # ignore/add/remove/force + +# Add or remove space after open brace in an unnamed temporary +# direct-list-initialization. +sp_after_type_brace_init_lst_open = ignore # ignore/add/remove/force + +# Add or remove space before close brace in an unnamed temporary +# direct-list-initialization. +sp_before_type_brace_init_lst_close = ignore # ignore/add/remove/force + +# Add or remove space inside an unnamed temporary direct-list-initialization. +sp_inside_type_brace_init_lst = ignore # ignore/add/remove/force + +# Add or remove space inside '{' and '}'. +sp_inside_braces = ignore # ignore/add/remove/force + +# Add or remove space inside '{}'. +sp_inside_braces_empty = ignore # ignore/add/remove/force + +# Add or remove space around trailing return operator '->'. +sp_trailing_return = ignore # ignore/add/remove/force + +# Add or remove space between return type and function name. A minimum of 1 +# is forced except for pointer return types. +sp_type_func = ignore # ignore/add/remove/force + +# Add or remove space between type and open brace of an unnamed temporary +# direct-list-initialization. +sp_type_brace_init_lst = ignore # ignore/add/remove/force + +# Add or remove space between function name and '(' on function declaration. +sp_func_proto_paren = ignore # ignore/add/remove/force + +# Add or remove space between function name and '()' on function declaration +# without parameters. +sp_func_proto_paren_empty = ignore # ignore/add/remove/force + +# Add or remove space between function name and '(' with a typedef specifier. +sp_func_type_paren = ignore # ignore/add/remove/force + +# Add or remove space between alias name and '(' of a non-pointer function type typedef. +sp_func_def_paren = ignore # ignore/add/remove/force + +# Add or remove space between function name and '()' on function definition +# without parameters. +sp_func_def_paren_empty = ignore # ignore/add/remove/force + +# Add or remove space inside empty function '()'. +# Overrides sp_after_angle unless use_sp_after_angle_always is set to true. +sp_inside_fparens = ignore # ignore/add/remove/force + +# Add or remove space inside function '(' and ')'. +sp_inside_fparen = ignore # ignore/add/remove/force + +# Add or remove space inside the first parentheses in a function type, as in +# 'void (*x)(...)'. +sp_inside_tparen = ignore # ignore/add/remove/force + +# Add or remove space between the ')' and '(' in a function type, as in +# 'void (*x)(...)'. +sp_after_tparen_close = ignore # ignore/add/remove/force + +# Add or remove space between ']' and '(' when part of a function call. +sp_square_fparen = ignore # ignore/add/remove/force + +# Add or remove space between ')' and '{' of function. +sp_fparen_brace = ignore # ignore/add/remove/force + +# Add or remove space between ')' and '{' of s function call in object +# initialization. +# +# Overrides sp_fparen_brace. +sp_fparen_brace_initializer = ignore # ignore/add/remove/force + +# (Java) Add or remove space between ')' and '{{' of double brace initializer. +sp_fparen_dbrace = ignore # ignore/add/remove/force + +# Add or remove space between function name and '(' on function calls. +sp_func_call_paren = ignore # ignore/add/remove/force + +# Add or remove space between function name and '()' on function calls without +# parameters. If set to ignore (the default), sp_func_call_paren is used. +sp_func_call_paren_empty = ignore # ignore/add/remove/force + +# Add or remove space between the user function name and '(' on function +# calls. You need to set a keyword to be a user function in the config file, +# like: +# set func_call_user tr _ i18n +sp_func_call_user_paren = ignore # ignore/add/remove/force + +# Add or remove space inside user function '(' and ')'. +sp_func_call_user_inside_fparen = ignore # ignore/add/remove/force + +# Add or remove space between nested parentheses with user functions, +# i.e. '((' vs. '( ('. +sp_func_call_user_paren_paren = ignore # ignore/add/remove/force + +# Add or remove space between a constructor/destructor and the open +# parenthesis. +sp_func_class_paren = ignore # ignore/add/remove/force + +# Add or remove space between a constructor without parameters or destructor +# and '()'. +sp_func_class_paren_empty = ignore # ignore/add/remove/force + +# Add or remove space between 'return' and '('. +sp_return_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'return' and '{'. +sp_return_brace = ignore # ignore/add/remove/force + +# Add or remove space between '__attribute__' and '('. +sp_attribute_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'defined' and '(' in '#if defined (FOO)'. +sp_defined_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'throw' and '(' in 'throw (something)'. +sp_throw_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'throw' and anything other than '(' as in +# '@throw [...];'. +sp_after_throw = ignore # ignore/add/remove/force + +# Add or remove space between 'catch' and '(' in 'catch (something) { }'. +# If set to ignore, sp_before_sparen is used. +sp_catch_paren = ignore # ignore/add/remove/force + +# (OC) Add or remove space between '@catch' and '(' +# in '@catch (something) { }'. If set to ignore, sp_catch_paren is used. +sp_oc_catch_paren = ignore # ignore/add/remove/force + +# (OC) Add or remove space between class name and '(' +# in '@interface className(categoryName):BaseClass' +sp_oc_classname_paren = ignore # ignore/add/remove/force + +# (D) Add or remove space between 'version' and '(' +# in 'version (something) { }'. If set to ignore, sp_before_sparen is used. +sp_version_paren = ignore # ignore/add/remove/force + +# (D) Add or remove space between 'scope' and '(' +# in 'scope (something) { }'. If set to ignore, sp_before_sparen is used. +sp_scope_paren = ignore # ignore/add/remove/force + +# Add or remove space between 'super' and '(' in 'super (something)'. +# +# Default: remove +sp_super_paren = remove # ignore/add/remove/force + +# Add or remove space between 'this' and '(' in 'this (something)'. +# +# Default: remove +sp_this_paren = remove # ignore/add/remove/force + +# Add or remove space between a macro name and its definition. +sp_macro = ignore # ignore/add/remove/force + +# Add or remove space between a macro function ')' and its definition. +sp_macro_func = ignore # ignore/add/remove/force + +# Add or remove space between 'else' and '{' if on the same line. +sp_else_brace = ignore # ignore/add/remove/force + +# Add or remove space between '}' and 'else' if on the same line. +sp_brace_else = ignore # ignore/add/remove/force + +# Add or remove space between '}' and the name of a typedef on the same line. +sp_brace_typedef = ignore # ignore/add/remove/force + +# Add or remove space before the '{' of a 'catch' statement, if the '{' and +# 'catch' are on the same line, as in 'catch (decl) {'. +sp_catch_brace = ignore # ignore/add/remove/force + +# (OC) Add or remove space before the '{' of a '@catch' statement, if the '{' +# and '@catch' are on the same line, as in '@catch (decl) {'. +# If set to ignore, sp_catch_brace is used. +sp_oc_catch_brace = ignore # ignore/add/remove/force + +# Add or remove space between '}' and 'catch' if on the same line. +sp_brace_catch = ignore # ignore/add/remove/force + +# (OC) Add or remove space between '}' and '@catch' if on the same line. +# If set to ignore, sp_brace_catch is used. +sp_oc_brace_catch = ignore # ignore/add/remove/force + +# Add or remove space between 'finally' and '{' if on the same line. +sp_finally_brace = ignore # ignore/add/remove/force + +# Add or remove space between '}' and 'finally' if on the same line. +sp_brace_finally = ignore # ignore/add/remove/force + +# Add or remove space between 'try' and '{' if on the same line. +sp_try_brace = ignore # ignore/add/remove/force + +# Add or remove space between get/set and '{' if on the same line. +sp_getset_brace = ignore # ignore/add/remove/force + +# Add or remove space between a variable and '{' for C++ uniform +# initialization. +# +# Default: add +sp_word_brace = add # ignore/add/remove/force + +# Add or remove space between a variable and '{' for a namespace. +# +# Default: add +sp_word_brace_ns = add # ignore/add/remove/force + +# Add or remove space before the '::' operator. +sp_before_dc = ignore # ignore/add/remove/force + +# Add or remove space after the '::' operator. +sp_after_dc = ignore # ignore/add/remove/force + +# (D) Add or remove around the D named array initializer ':' operator. +sp_d_array_colon = ignore # ignore/add/remove/force + +# Add or remove space after the '!' (not) unary operator. +# +# Default: remove +sp_not = remove # ignore/add/remove/force + +# Add or remove space after the '~' (invert) unary operator. +# +# Default: remove +sp_inv = remove # ignore/add/remove/force + +# Add or remove space after the '&' (address-of) unary operator. This does not +# affect the spacing after a '&' that is part of a type. +# +# Default: remove +sp_addr = remove # ignore/add/remove/force + +# Add or remove space around the '.' or '->' operators. +# +# Default: remove +sp_member = remove # ignore/add/remove/force + +# Add or remove space after the '*' (dereference) unary operator. This does +# not affect the spacing after a '*' that is part of a type. +# +# Default: remove +sp_deref = remove # ignore/add/remove/force + +# Add or remove space after '+' or '-', as in 'x = -5' or 'y = +7'. +# +# Default: remove +sp_sign = remove # ignore/add/remove/force + +# Add or remove space between '++' and '--' the word to which it is being +# applied, as in '(--x)' or 'y++;'. +# +# Default: remove +sp_incdec = remove # ignore/add/remove/force + +# Add or remove space before a backslash-newline at the end of a line. +# +# Default: add +sp_before_nl_cont = add # ignore/add/remove/force + +# (OC) Add or remove space after the scope '+' or '-', as in '-(void) foo;' +# or '+(int) bar;'. +sp_after_oc_scope = ignore # ignore/add/remove/force + +# (OC) Add or remove space after the colon in message specs, +# i.e. '-(int) f:(int) x;' vs. '-(int) f: (int) x;'. +sp_after_oc_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space before the colon in message specs, +# i.e. '-(int) f: (int) x;' vs. '-(int) f : (int) x;'. +sp_before_oc_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space after the colon in immutable dictionary expression +# 'NSDictionary *test = @{@"foo" :@"bar"};'. +sp_after_oc_dict_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space before the colon in immutable dictionary expression +# 'NSDictionary *test = @{@"foo" :@"bar"};'. +sp_before_oc_dict_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space after the colon in message specs, +# i.e. '[object setValue:1];' vs. '[object setValue: 1];'. +sp_after_send_oc_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space before the colon in message specs, +# i.e. '[object setValue:1];' vs. '[object setValue :1];'. +sp_before_send_oc_colon = ignore # ignore/add/remove/force + +# (OC) Add or remove space after the (type) in message specs, +# i.e. '-(int)f: (int) x;' vs. '-(int)f: (int)x;'. +sp_after_oc_type = ignore # ignore/add/remove/force + +# (OC) Add or remove space after the first (type) in message specs, +# i.e. '-(int) f:(int)x;' vs. '-(int)f:(int)x;'. +sp_after_oc_return_type = ignore # ignore/add/remove/force + +# (OC) Add or remove space between '@selector' and '(', +# i.e. '@selector(msgName)' vs. '@selector (msgName)'. +# Also applies to '@protocol()' constructs. +sp_after_oc_at_sel = ignore # ignore/add/remove/force + +# (OC) Add or remove space between '@selector(x)' and the following word, +# i.e. '@selector(foo) a:' vs. '@selector(foo)a:'. +sp_after_oc_at_sel_parens = ignore # ignore/add/remove/force + +# (OC) Add or remove space inside '@selector' parentheses, +# i.e. '@selector(foo)' vs. '@selector( foo )'. +# Also applies to '@protocol()' constructs. +sp_inside_oc_at_sel_parens = ignore # ignore/add/remove/force + +# (OC) Add or remove space before a block pointer caret, +# i.e. '^int (int arg){...}' vs. ' ^int (int arg){...}'. +sp_before_oc_block_caret = ignore # ignore/add/remove/force + +# (OC) Add or remove space after a block pointer caret, +# i.e. '^int (int arg){...}' vs. '^ int (int arg){...}'. +sp_after_oc_block_caret = ignore # ignore/add/remove/force + +# (OC) Add or remove space between the receiver and selector in a message, +# as in '[receiver selector ...]'. +sp_after_oc_msg_receiver = ignore # ignore/add/remove/force + +# (OC) Add or remove space after '@property'. +sp_after_oc_property = ignore # ignore/add/remove/force + +# (OC) Add or remove space between '@synchronized' and the open parenthesis, +# i.e. '@synchronized(foo)' vs. '@synchronized (foo)'. +sp_after_oc_synchronized = ignore # ignore/add/remove/force + +# Add or remove space around the ':' in 'b ? t : f'. +sp_cond_colon = ignore # ignore/add/remove/force + +# Add or remove space before the ':' in 'b ? t : f'. +# +# Overrides sp_cond_colon. +sp_cond_colon_before = ignore # ignore/add/remove/force + +# Add or remove space after the ':' in 'b ? t : f'. +# +# Overrides sp_cond_colon. +sp_cond_colon_after = ignore # ignore/add/remove/force + +# Add or remove space around the '?' in 'b ? t : f'. +sp_cond_question = ignore # ignore/add/remove/force + +# Add or remove space before the '?' in 'b ? t : f'. +# +# Overrides sp_cond_question. +sp_cond_question_before = ignore # ignore/add/remove/force + +# Add or remove space after the '?' in 'b ? t : f'. +# +# Overrides sp_cond_question. +sp_cond_question_after = ignore # ignore/add/remove/force + +# In the abbreviated ternary form '(a ?: b)', add or remove space between '?' +# and ':'. +# +# Overrides all other sp_cond_* options. +sp_cond_ternary_short = ignore # ignore/add/remove/force + +# Fix the spacing between 'case' and the label. Only 'ignore' and 'force' make +# sense here. +sp_case_label = ignore # ignore/add/remove/force + +# (D) Add or remove space around the D '..' operator. +sp_range = ignore # ignore/add/remove/force + +# Add or remove space after ':' in a Java/C++11 range-based 'for', +# as in 'for (Type var : expr)'. +sp_after_for_colon = ignore # ignore/add/remove/force + +# Add or remove space before ':' in a Java/C++11 range-based 'for', +# as in 'for (Type var : expr)'. +sp_before_for_colon = ignore # ignore/add/remove/force + +# (D) Add or remove space between 'extern' and '(' as in 'extern (C)'. +sp_extern_paren = ignore # ignore/add/remove/force + +# Add or remove space after the opening of a C++ comment, +# i.e. '// A' vs. '//A'. +sp_cmt_cpp_start = ignore # ignore/add/remove/force + +# If true, space is added with sp_cmt_cpp_start will be added after doxygen +# sequences like '///', '///<', '//!' and '//!<'. +sp_cmt_cpp_doxygen = false # true/false + +# If true, space is added with sp_cmt_cpp_start will be added after Qt +# translator or meta-data comments like '//:', '//=', and '//~'. +sp_cmt_cpp_qttr = false # true/false + +# Add or remove space between #else or #endif and a trailing comment. +sp_endif_cmt = ignore # ignore/add/remove/force + +# Add or remove space after 'new', 'delete' and 'delete[]'. +sp_after_new = ignore # ignore/add/remove/force + +# Add or remove space between 'new' and '(' in 'new()'. +sp_between_new_paren = ignore # ignore/add/remove/force + +# Add or remove space between ')' and type in 'new(foo) BAR'. +sp_after_newop_paren = ignore # ignore/add/remove/force + +# Add or remove space inside parenthesis of the new operator +# as in 'new(foo) BAR'. +sp_inside_newop_paren = ignore # ignore/add/remove/force + +# Add or remove space after the open parenthesis of the new operator, +# as in 'new(foo) BAR'. +# +# Overrides sp_inside_newop_paren. +sp_inside_newop_paren_open = ignore # ignore/add/remove/force + +# Add or remove space before the close parenthesis of the new operator, +# as in 'new(foo) BAR'. +# +# Overrides sp_inside_newop_paren. +sp_inside_newop_paren_close = ignore # ignore/add/remove/force + +# Add or remove space before a trailing or embedded comment. +sp_before_tr_emb_cmt = ignore # ignore/add/remove/force + +# Number of spaces before a trailing or embedded comment. +sp_num_before_tr_emb_cmt = 0 # unsigned number + +# (Java) Add or remove space between an annotation and the open parenthesis. +sp_annotation_paren = ignore # ignore/add/remove/force + +# If true, vbrace tokens are dropped to the previous token and skipped. +sp_skip_vbrace_tokens = false # true/false + +# Add or remove space after 'noexcept'. +sp_after_noexcept = ignore # ignore/add/remove/force + +# Add or remove space after '_'. +sp_vala_after_translation = ignore # ignore/add/remove/force + +# If true, a is inserted after #define. +force_tab_after_define = false # true/false + +# +# Indenting options +# + +# The number of columns to indent per level. Usually 2, 3, 4, or 8. +# +# Default: 8 +indent_columns = 4 # unsigned number + +# The continuation indent. If non-zero, this overrides the indent of '(', '[' +# and '=' continuation indents. Negative values are OK; negative value is +# absolute and not increased for each '(' or '[' level. +# +# For FreeBSD, this is set to 4. +indent_continue = 0 # number + +# The continuation indent, only for class header line(s). If non-zero, this +# overrides the indent of 'class' continuation indents. +indent_continue_class_head = 0 # unsigned number + +# Whether to indent empty lines (i.e. lines which contain only spaces before +# the newline character). +indent_single_newlines = false # true/false + +# The continuation indent for func_*_param if they are true. If non-zero, this +# overrides the indent. +indent_param = 0 # unsigned number + +# How to use tabs when indenting code. +# +# 0: Spaces only +# 1: Indent with tabs to brace level, align with spaces (default) +# 2: Indent and align with tabs, using spaces when not on a tabstop +# +# Default: 1 +indent_with_tabs = 0 # unsigned number + +# Whether to indent comments that are not at a brace level with tabs on a +# tabstop. Requires indent_with_tabs=2. If false, will use spaces. +indent_cmt_with_tabs = false # true/false + +# Whether to indent strings broken by '\' so that they line up. +indent_align_string = false # true/false + +# The number of spaces to indent multi-line XML strings. +# Requires indent_align_string=true. +indent_xml_string = 0 # unsigned number + +# Spaces to indent '{' from level. +indent_brace = 0 # unsigned number + +# Whether braces are indented to the body level. +indent_braces = false # true/false + +# Whether to disable indenting function braces if indent_braces=true. +indent_braces_no_func = false # true/false + +# Whether to disable indenting class braces if indent_braces=true. +indent_braces_no_class = false # true/false + +# Whether to disable indenting struct braces if indent_braces=true. +indent_braces_no_struct = false # true/false + +# Whether to indent based on the size of the brace parent, +# i.e. 'if' => 3 spaces, 'for' => 4 spaces, etc. +indent_brace_parent = false # true/false + +# Whether to indent based on the open parenthesis instead of the open brace +# in '({\n'. +indent_paren_open_brace = false # true/false + +# (C#) Whether to indent the brace of a C# delegate by another level. +indent_cs_delegate_brace = false # true/false + +# (C#) Whether to indent a C# delegate (to handle delegates with no brace) by +# another level. +indent_cs_delegate_body = false # true/false + +# Whether to indent the body of a 'namespace'. +indent_namespace = false # true/false + +# Whether to indent only the first namespace, and not any nested namespaces. +# Requires indent_namespace=true. +indent_namespace_single_indent = false # true/false + +# The number of spaces to indent a namespace block. +# If set to zero, use the value indent_columns +indent_namespace_level = 0 # unsigned number + +# If the body of the namespace is longer than this number, it won't be +# indented. Requires indent_namespace=true. 0 means no limit. +indent_namespace_limit = 0 # unsigned number + +# Whether the 'extern "C"' body is indented. +indent_extern = false # true/false + +# Whether the 'class' body is indented. +indent_class = true # true/false + +# Whether to indent the stuff after a leading base class colon. +indent_class_colon = false # true/false + +# Whether to indent based on a class colon instead of the stuff after the +# colon. Requires indent_class_colon=true. +indent_class_on_colon = false # true/false + +# Whether to indent the stuff after a leading class initializer colon. +indent_constr_colon = false # true/false + +# Virtual indent from the ':' for member initializers. +# +# Default: 2 +indent_ctor_init_leading = 2 # unsigned number + +# Additional indent for constructor initializer list. +# Negative values decrease indent down to the first column. +indent_ctor_init = 0 # number + +# Whether to indent 'if' following 'else' as a new block under the 'else'. +# If false, 'else\nif' is treated as 'else if' for indenting purposes. +indent_else_if = false # true/false + +# Amount to indent variable declarations after a open brace. +# +# <0: Relative +# >=0: Absolute +indent_var_def_blk = 0 # number + +# Whether to indent continued variable declarations instead of aligning. +indent_var_def_cont = false # true/false + +# Whether to indent continued shift expressions ('<<' and '>>') instead of +# aligning. Set align_left_shift=false when enabling this. +indent_shift = false # true/false + +# Whether to force indentation of function definitions to start in column 1. +indent_func_def_force_col1 = false # true/false + +# Whether to indent continued function call parameters one indent level, +# rather than aligning parameters under the open parenthesis. +indent_func_call_param = true # true/false + +# Whether to indent continued function definition parameters one indent level, +# rather than aligning parameters under the open parenthesis. +indent_func_def_param = true # true/false + +# for function definitions, only if indent_func_def_param is false +# Allows to align params when appropriate and indent them when not +# behave as if it was true if paren position is more than this value +# if paren position is more than the option value +indent_func_def_param_paren_pos_threshold = 0 # unsigned number + +# Whether to indent continued function call prototype one indent level, +# rather than aligning parameters under the open parenthesis. +indent_func_proto_param = true # true/false + +# Whether to indent continued function call declaration one indent level, +# rather than aligning parameters under the open parenthesis. +indent_func_class_param = true # true/false + +# Whether to indent continued class variable constructors one indent level, +# rather than aligning parameters under the open parenthesis. +indent_func_ctor_var_param = true # true/false + +# Whether to indent continued template parameter list one indent level, +# rather than aligning parameters under the open parenthesis. +indent_template_param = true # true/false + +# Double the indent for indent_func_xxx_param options. +# Use both values of the options indent_columns and indent_param. +indent_func_param_double = false # true/false + +# Indentation column for standalone 'const' qualifier on a function +# prototype. +indent_func_const = 0 # unsigned number + +# Indentation column for standalone 'throw' qualifier on a function +# prototype. +indent_func_throw = 0 # unsigned number + +# How to indent within a macro followed by a brace on the same line +# This allows reducing the indent in macros that have (for example) +# `do { ... } while (0)` blocks bracketing them. +# +# true: add an indent for the brace on the same line as the macro +# false: do not add an indent for the brace on the same line as the macro +# +# Default: true +indent_macro_brace = true # true/false + +# The number of spaces to indent a continued '->' or '.'. +# Usually set to 0, 1, or indent_columns. +indent_member = 0 # unsigned number + +# Whether lines broken at '.' or '->' should be indented by a single indent. +# The indent_member option will not be effective if this is set to true. +indent_member_single = false # true/false + +# Spaces to indent single line ('//') comments on lines before code. +indent_sing_line_comments = 0 # unsigned number + +# Whether to indent trailing single line ('//') comments relative to the code +# instead of trying to keep the same absolute column. +indent_relative_single_line_comments = false # true/false + +# Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns. +indent_switch_case = 0 # unsigned number + +# indent 'break' with 'case' from 'switch'. +indent_switch_break_with_case = false # true/false + +# Whether to indent preprocessor statements inside of switch statements. +# +# Default: true +indent_switch_pp = true # true/false + +# Spaces to shift the 'case' line, without affecting any other lines. +# Usually 0. +indent_case_shift = 0 # unsigned number + +# Spaces to indent '{' from 'case'. By default, the brace will appear under +# the 'c' in case. Usually set to 0 or indent_columns. Negative values are OK. +indent_case_brace = 0 # number + +# Whether to indent comments found in first column. +indent_col1_comment = false # true/false + +# Whether to indent multi string literal in first column. +indent_col1_multi_string_literal = false # true/false + +# How to indent goto labels. +# +# >0: Absolute column where 1 is the leftmost column +# <=0: Subtract from brace indent +# +# Default: 1 +indent_label = 1 # number + +# How to indent access specifiers that are followed by a +# colon. +# +# >0: Absolute column where 1 is the leftmost column +# <=0: Subtract from brace indent +# +# Default: 1 +indent_access_spec = 1 # number + +# Whether to indent the code after an access specifier by one level. +# If true, this option forces 'indent_access_spec=0'. +indent_access_spec_body = false # true/false + +# If an open parenthesis is followed by a newline, whether to indent the next +# line so that it lines up after the open parenthesis (not recommended). +indent_paren_nl = false # true/false + +# How to indent a close parenthesis after a newline. +# +# 0: Indent to body level (default) +# 1: Align under the open parenthesis +# 2: Indent to the brace level +indent_paren_close = 0 # unsigned number + +# Whether to indent the open parenthesis of a function definition, +# if the parenthesis is on its own line. +indent_paren_after_func_def = false # true/false + +# Whether to indent the open parenthesis of a function declaration, +# if the parenthesis is on its own line. +indent_paren_after_func_decl = false # true/false + +# Whether to indent the open parenthesis of a function call, +# if the parenthesis is on its own line. +indent_paren_after_func_call = false # true/false + +# Whether to indent a comma when inside a parenthesis. +# If true, aligns under the open parenthesis. +indent_comma_paren = false # true/false + +# Whether to indent a Boolean operator when inside a parenthesis. +# If true, aligns under the open parenthesis. +indent_bool_paren = false # true/false + +# Whether to indent a semicolon when inside a for parenthesis. +# If true, aligns under the open for parenthesis. +indent_semicolon_for_paren = false # true/false + +# Whether to align the first expression to following ones +# if indent_bool_paren=true. +indent_first_bool_expr = false # true/false + +# Whether to align the first expression to following ones +# if indent_semicolon_for_paren=true. +indent_first_for_expr = false # true/false + +# If an open square is followed by a newline, whether to indent the next line +# so that it lines up after the open square (not recommended). +indent_square_nl = false # true/false + +# (ESQL/C) Whether to preserve the relative indent of 'EXEC SQL' bodies. +indent_preserve_sql = false # true/false + +# Whether to align continued statements at the '='. If false or if the '=' is +# followed by a newline, the next line is indent one tab. +# +# Default: true +indent_align_assign = true # true/false + +# Whether to align continued statements at the '('. If false or the '(' is +# followed by a newline, the next line indent is one tab. +# +# Default: true +indent_align_paren = true # true/false + +# (OC) Whether to indent Objective-C blocks at brace level instead of usual +# rules. +indent_oc_block = false # true/false + +# (OC) Indent for Objective-C blocks in a message relative to the parameter +# name. +# +# =0: Use indent_oc_block rules +# >0: Use specified number of spaces to indent +indent_oc_block_msg = 0 # unsigned number + +# (OC) Minimum indent for subsequent parameters +indent_oc_msg_colon = 0 # unsigned number + +# (OC) Whether to prioritize aligning with initial colon (and stripping spaces +# from lines, if necessary). +# +# Default: true +indent_oc_msg_prioritize_first_colon = true # true/false + +# (OC) Whether to indent blocks the way that Xcode does by default +# (from the keyword if the parameter is on its own line; otherwise, from the +# previous indentation level). Requires indent_oc_block_msg=true. +indent_oc_block_msg_xcode_style = false # true/false + +# (OC) Whether to indent blocks from where the brace is, relative to a +# message keyword. Requires indent_oc_block_msg=true. +indent_oc_block_msg_from_keyword = false # true/false + +# (OC) Whether to indent blocks from where the brace is, relative to a message +# colon. Requires indent_oc_block_msg=true. +indent_oc_block_msg_from_colon = false # true/false + +# (OC) Whether to indent blocks from where the block caret is. +# Requires indent_oc_block_msg=true. +indent_oc_block_msg_from_caret = false # true/false + +# (OC) Whether to indent blocks from where the brace caret is. +# Requires indent_oc_block_msg=true. +indent_oc_block_msg_from_brace = false # true/false + +# When indenting after virtual brace open and newline add further spaces to +# reach this minimum indent. +indent_min_vbrace_open = 0 # unsigned number + +# Whether to add further spaces after regular indent to reach next tabstop +# when identing after virtual brace open and newline. +indent_vbrace_open_on_tabstop = false # true/false + +# How to indent after a brace followed by another token (not a newline). +# true: indent all contained lines to match the token +# false: indent all contained lines to match the brace +# +# Default: true +indent_token_after_brace = true # true/false + +# Whether to indent the body of a C++11 lambda. +indent_cpp_lambda_body = false # true/false + +# (C#) Whether to indent a 'using' block if no braces are used. +# +# Default: true +indent_using_block = true # true/false + +# How to indent the continuation of ternary operator. +# +# 0: Off (default) +# 1: When the `if_false` is a continuation, indent it under `if_false` +# 2: When the `:` is a continuation, indent it under `?` +indent_ternary_operator = 0 # unsigned number + +# If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column. +indent_off_after_return_new = false # true/false + +# If true, the tokens after return are indented with regular single indentation. By default (false) the indentation is after the return token. +indent_single_after_return = false # true/false + +# Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they +# have their own indentation). +indent_ignore_asm_block = false # true/false + +# +# Newline adding and removing options +# + +# Whether to collapse empty blocks between '{' and '}'. +nl_collapse_empty_body = false # true/false + +# Don't split one-line braced assignments, as in 'foo_t f = { 1, 2 };'. +nl_assign_leave_one_liners = true # true/false + +# Don't split one-line braced statements inside a 'class xx { }' body. +nl_class_leave_one_liners = true # true/false + +# Don't split one-line enums, as in 'enum foo { BAR = 15 };' +nl_enum_leave_one_liners = true # true/false + +# Don't split one-line get or set functions. +nl_getset_leave_one_liners = true # true/false + +# (C#) Don't split one-line property get or set functions. +nl_cs_property_leave_one_liners = true # true/false + +# Don't split one-line function definitions, as in 'int foo() { return 0; }'. +# night modify nl_func_type_name +nl_func_leave_one_liners = true # true/false + +# Don't split one-line C++11 lambdas, as in '[]() { return 0; }'. +nl_cpp_lambda_leave_one_liners = false # true/false + +# Don't split one-line if/else statements, as in 'if(...) b++;'. +nl_if_leave_one_liners = true # true/false + +# Don't split one-line while statements, as in 'while(...) b++;'. +nl_while_leave_one_liners = true # true/false + +# Don't split one-line for statements, as in 'for(...) b++;'. +nl_for_leave_one_liners = true # true/false + +# (OC) Don't split one-line Objective-C messages. +nl_oc_msg_leave_one_liner = false # true/false + +# (OC) Add or remove newline between method declaration and '{'. +nl_oc_mdef_brace = ignore # ignore/add/remove/force + +# (OC) Add or remove newline between Objective-C block signature and '{'. +nl_oc_block_brace = ignore # ignore/add/remove/force + +# (OC) Add or remove blank line before '@interface' statement. +nl_oc_before_interface = ignore # ignore/add/remove/force + +# (OC) Add or remove blank line before '@implementation' statement. +nl_oc_before_implementation = ignore # ignore/add/remove/force + +# (OC) Add or remove blank line before '@end' statement. +nl_oc_before_end = ignore # ignore/add/remove/force + +# (OC) Add or remove newline between '@interface' and '{'. +nl_oc_interface_brace = ignore # ignore/add/remove/force + +# (OC) Add or remove newline between '@implementation' and '{'. +nl_oc_implementation_brace = ignore # ignore/add/remove/force + +# Add or remove newlines at the start of the file. +nl_start_of_file = ignore # ignore/add/remove/force + +# The minimum number of newlines at the start of the file (only used if +# nl_start_of_file is 'add' or 'force'). +nl_start_of_file_min = 0 # unsigned number + +# Add or remove newline at the end of the file. +nl_end_of_file = ignore # ignore/add/remove/force + +# The minimum number of newlines at the end of the file (only used if +# nl_end_of_file is 'add' or 'force'). +nl_end_of_file_min = 0 # unsigned number + +# Add or remove newline between '=' and '{'. +nl_assign_brace = ignore # ignore/add/remove/force + +# (D) Add or remove newline between '=' and '['. +nl_assign_square = ignore # ignore/add/remove/force + +# Add or remove newline between '[]' and '{'. +nl_tsquare_brace = ignore # ignore/add/remove/force + +# (D) Add or remove newline after '= ['. Will also affect the newline before +# the ']'. +nl_after_square_assign = ignore # ignore/add/remove/force + +# Add or remove newline between a function call's ')' and '{', as in +# 'list_for_each(item, &list) { }'. +nl_fcall_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'enum' and '{'. +nl_enum_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'enum' and 'class'. +nl_enum_class = ignore # ignore/add/remove/force + +# Add or remove newline between 'enum class' and the identifier. +nl_enum_class_identifier = ignore # ignore/add/remove/force + +# Add or remove newline between 'enum class' type and ':'. +nl_enum_identifier_colon = ignore # ignore/add/remove/force + +# Add or remove newline between 'enum class identifier :' and type. +nl_enum_colon_type = ignore # ignore/add/remove/force + +# Add or remove newline between 'struct and '{'. +nl_struct_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'union' and '{'. +nl_union_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'if' and '{'. +nl_if_brace = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and 'else'. +nl_brace_else = ignore # ignore/add/remove/force + +# Add or remove newline between 'else if' and '{'. If set to ignore, +# nl_if_brace is used instead. +nl_elseif_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'else' and '{'. +nl_else_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'else' and 'if'. +nl_else_if = ignore # ignore/add/remove/force + +# Add or remove newline before 'if'/'else if' closing parenthesis. +nl_before_if_closing_paren = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and 'finally'. +nl_brace_finally = ignore # ignore/add/remove/force + +# Add or remove newline between 'finally' and '{'. +nl_finally_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'try' and '{'. +nl_try_brace = ignore # ignore/add/remove/force + +# Add or remove newline between get/set and '{'. +nl_getset_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'for' and '{'. +nl_for_brace = ignore # ignore/add/remove/force + +# Add or remove newline before the '{' of a 'catch' statement, as in +# 'catch (decl) {'. +nl_catch_brace = ignore # ignore/add/remove/force + +# (OC) Add or remove newline before the '{' of a '@catch' statement, as in +# '@catch (decl) {'. If set to ignore, nl_catch_brace is used. +nl_oc_catch_brace = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and 'catch'. +nl_brace_catch = ignore # ignore/add/remove/force + +# (OC) Add or remove newline between '}' and '@catch'. If set to ignore, +# nl_brace_catch is used. +nl_oc_brace_catch = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and ']'. +nl_brace_square = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and ')' in a function invocation. +nl_brace_fparen = ignore # ignore/add/remove/force + +# Add or remove newline between 'while' and '{'. +nl_while_brace = ignore # ignore/add/remove/force + +# (D) Add or remove newline between 'scope (x)' and '{'. +nl_scope_brace = ignore # ignore/add/remove/force + +# (D) Add or remove newline between 'unittest' and '{'. +nl_unittest_brace = ignore # ignore/add/remove/force + +# (D) Add or remove newline between 'version (x)' and '{'. +nl_version_brace = ignore # ignore/add/remove/force + +# (C#) Add or remove newline between 'using' and '{'. +nl_using_brace = ignore # ignore/add/remove/force + +# Add or remove newline between two open or close braces. Due to general +# newline/brace handling, REMOVE may not work. +nl_brace_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'do' and '{'. +nl_do_brace = ignore # ignore/add/remove/force + +# Add or remove newline between '}' and 'while' of 'do' statement. +nl_brace_while = ignore # ignore/add/remove/force + +# Add or remove newline between 'switch' and '{'. +nl_switch_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'synchronized' and '{'. +nl_synchronized_brace = ignore # ignore/add/remove/force + +# Add a newline between ')' and '{' if the ')' is on a different line than the +# if/for/etc. +# +# Overrides nl_for_brace, nl_if_brace, nl_switch_brace, nl_while_switch and +# nl_catch_brace. +nl_multi_line_cond = false # true/false + +# Add a newline after '(' if an if/for/while/switch condition spans multiple +# lines +nl_multi_line_sparen_open = ignore # ignore/add/remove/force + +# Add a newline before ')' if an if/for/while/switch condition spans multiple +# lines. Overrides nl_before_if_closing_paren if both are specified. +nl_multi_line_sparen_close = ignore # ignore/add/remove/force + +# Force a newline in a define after the macro name for multi-line defines. +nl_multi_line_define = false # true/false + +# Whether to add a newline before 'case', and a blank line before a 'case' +# statement that follows a ';' or '}'. +nl_before_case = false # true/false + +# Whether to add a newline after a 'case' statement. +nl_after_case = false # true/false + +# Add or remove newline between a case ':' and '{'. +# +# Overrides nl_after_case. +nl_case_colon_brace = ignore # ignore/add/remove/force + +# Add or remove newline between ')' and 'throw'. +nl_before_throw = ignore # ignore/add/remove/force + +# Add or remove newline between 'namespace' and '{'. +nl_namespace_brace = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template class. +nl_template_class = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template class declaration. +# +# Overrides nl_template_class. +nl_template_class_decl = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<>' of a specialized class declaration. +# +# Overrides nl_template_class_decl. +nl_template_class_decl_special = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template class definition. +# +# Overrides nl_template_class. +nl_template_class_def = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<>' of a specialized class definition. +# +# Overrides nl_template_class_def. +nl_template_class_def_special = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template function. +nl_template_func = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template function +# declaration. +# +# Overrides nl_template_func. +nl_template_func_decl = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<>' of a specialized function +# declaration. +# +# Overrides nl_template_func_decl. +nl_template_func_decl_special = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template function +# definition. +# +# Overrides nl_template_func. +nl_template_func_def = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<>' of a specialized function +# definition. +# +# Overrides nl_template_func_def. +nl_template_func_def_special = ignore # ignore/add/remove/force + +# Add or remove newline after 'template<...>' of a template variable. +nl_template_var = ignore # ignore/add/remove/force + +# Add or remove newline between 'template<...>' and 'using' of a templated +# type alias. +nl_template_using = ignore # ignore/add/remove/force + +# Add or remove newline between 'class' and '{'. +nl_class_brace = ignore # ignore/add/remove/force + +# Add or remove newline before or after (depending on pos_class_comma, +# may not be IGNORE) each',' in the base class list. +nl_class_init_args = ignore # ignore/add/remove/force + +# Add or remove newline after each ',' in the constructor member +# initialization. Related to nl_constr_colon, pos_constr_colon and +# pos_constr_comma. +nl_constr_init_args = ignore # ignore/add/remove/force + +# Add or remove newline before first element, after comma, and after last +# element, in 'enum'. +nl_enum_own_lines = ignore # ignore/add/remove/force + +# Add or remove newline between return type and function name in a function +# definition. +# might be modified by nl_func_leave_one_liners +nl_func_type_name = ignore # ignore/add/remove/force + +# Add or remove newline between return type and function name inside a class +# definition. If set to ignore, nl_func_type_name or nl_func_proto_type_name +# is used instead. +nl_func_type_name_class = ignore # ignore/add/remove/force + +# Add or remove newline between class specification and '::' +# in 'void A::f() { }'. Only appears in separate member implementation (does +# not appear with in-line implementation). +nl_func_class_scope = ignore # ignore/add/remove/force + +# Add or remove newline between function scope and name, as in +# 'void A :: f() { }'. +nl_func_scope_name = ignore # ignore/add/remove/force + +# Add or remove newline between return type and function name in a prototype. +nl_func_proto_type_name = ignore # ignore/add/remove/force + +# Add or remove newline between a function name and the opening '(' in the +# declaration. +nl_func_paren = ignore # ignore/add/remove/force + +# Overrides nl_func_paren for functions with no parameters. +nl_func_paren_empty = ignore # ignore/add/remove/force + +# Add or remove newline between a function name and the opening '(' in the +# definition. +nl_func_def_paren = ignore # ignore/add/remove/force + +# Overrides nl_func_def_paren for functions with no parameters. +nl_func_def_paren_empty = ignore # ignore/add/remove/force + +# Add or remove newline between a function name and the opening '(' in the +# call. +nl_func_call_paren = ignore # ignore/add/remove/force + +# Overrides nl_func_call_paren for functions with no parameters. +nl_func_call_paren_empty = ignore # ignore/add/remove/force + +# Add or remove newline after '(' in a function declaration. +nl_func_decl_start = ignore # ignore/add/remove/force + +# Add or remove newline after '(' in a function definition. +nl_func_def_start = ignore # ignore/add/remove/force + +# Overrides nl_func_decl_start when there is only one parameter. +nl_func_decl_start_single = ignore # ignore/add/remove/force + +# Overrides nl_func_def_start when there is only one parameter. +nl_func_def_start_single = ignore # ignore/add/remove/force + +# Whether to add a newline after '(' in a function declaration if '(' and ')' +# are in different lines. If false, nl_func_decl_start is used instead. +nl_func_decl_start_multi_line = false # true/false + +# Whether to add a newline after '(' in a function definition if '(' and ')' +# are in different lines. If false, nl_func_def_start is used instead. +nl_func_def_start_multi_line = false # true/false + +# Add or remove newline after each ',' in a function declaration. +nl_func_decl_args = ignore # ignore/add/remove/force + +# Add or remove newline after each ',' in a function definition. +nl_func_def_args = ignore # ignore/add/remove/force + +# Whether to add a newline after each ',' in a function declaration if '(' +# and ')' are in different lines. If false, nl_func_decl_args is used instead. +nl_func_decl_args_multi_line = false # true/false + +# Whether to add a newline after each ',' in a function definition if '(' +# and ')' are in different lines. If false, nl_func_def_args is used instead. +nl_func_def_args_multi_line = false # true/false + +# Add or remove newline before the ')' in a function declaration. +nl_func_decl_end = ignore # ignore/add/remove/force + +# Add or remove newline before the ')' in a function definition. +nl_func_def_end = ignore # ignore/add/remove/force + +# Overrides nl_func_decl_end when there is only one parameter. +nl_func_decl_end_single = ignore # ignore/add/remove/force + +# Overrides nl_func_def_end when there is only one parameter. +nl_func_def_end_single = ignore # ignore/add/remove/force + +# Whether to add a newline before ')' in a function declaration if '(' and ')' +# are in different lines. If false, nl_func_decl_end is used instead. +nl_func_decl_end_multi_line = false # true/false + +# Whether to add a newline before ')' in a function definition if '(' and ')' +# are in different lines. If false, nl_func_def_end is used instead. +nl_func_def_end_multi_line = false # true/false + +# Add or remove newline between '()' in a function declaration. +nl_func_decl_empty = ignore # ignore/add/remove/force + +# Add or remove newline between '()' in a function definition. +nl_func_def_empty = ignore # ignore/add/remove/force + +# Add or remove newline between '()' in a function call. +nl_func_call_empty = ignore # ignore/add/remove/force + +# Whether to add a newline after '(' in a function call, +# has preference over nl_func_call_start_multi_line. +nl_func_call_start = ignore # ignore/add/remove/force + +# Whether to add a newline after '(' in a function call if '(' and ')' are in +# different lines. +nl_func_call_start_multi_line = false # true/false + +# Whether to add a newline after each ',' in a function call if '(' and ')' +# are in different lines. +nl_func_call_args_multi_line = false # true/false + +# Whether to add a newline before ')' in a function call if '(' and ')' are in +# different lines. +nl_func_call_end_multi_line = false # true/false + +# Whether to add a newline after '<' of a template parameter list. +nl_template_start = false # true/false + +# Whether to add a newline after each ',' in a template parameter list. +nl_template_args = false # true/false + +# Whether to add a newline before '>' of a template parameter list. +nl_template_end = false # true/false + +# (OC) Whether to put each Objective-C message parameter on a separate line. +# See nl_oc_msg_leave_one_liner. +nl_oc_msg_args = false # true/false + +# Add or remove newline between function signature and '{'. +nl_fdef_brace = ignore # ignore/add/remove/force + +# Add or remove newline between function signature and '{', +# if signature ends with ')'. Overrides nl_fdef_brace. +nl_fdef_brace_cond = ignore # ignore/add/remove/force + +# Add or remove newline between C++11 lambda signature and '{'. +nl_cpp_ldef_brace = ignore # ignore/add/remove/force + +# Add or remove newline between 'return' and the return expression. +nl_return_expr = ignore # ignore/add/remove/force + +# Whether to add a newline after semicolons, except in 'for' statements. +nl_after_semicolon = false # true/false + +# (Java) Add or remove newline between the ')' and '{{' of the double brace +# initializer. +nl_paren_dbrace_open = ignore # ignore/add/remove/force + +# Whether to add a newline after the type in an unnamed temporary +# direct-list-initialization. +nl_type_brace_init_lst = ignore # ignore/add/remove/force + +# Whether to add a newline after the open brace in an unnamed temporary +# direct-list-initialization. +nl_type_brace_init_lst_open = ignore # ignore/add/remove/force + +# Whether to add a newline before the close brace in an unnamed temporary +# direct-list-initialization. +nl_type_brace_init_lst_close = ignore # ignore/add/remove/force + +# Whether to add a newline after '{'. This also adds a newline before the +# matching '}'. +nl_after_brace_open = false # true/false + +# Whether to add a newline between the open brace and a trailing single-line +# comment. Requires nl_after_brace_open=true. +nl_after_brace_open_cmt = false # true/false + +# Whether to add a newline after a virtual brace open with a non-empty body. +# These occur in un-braced if/while/do/for statement bodies. +nl_after_vbrace_open = false # true/false + +# Whether to add a newline after a virtual brace open with an empty body. +# These occur in un-braced if/while/do/for statement bodies. +nl_after_vbrace_open_empty = false # true/false + +# Whether to add a newline after '}'. Does not apply if followed by a +# necessary ';'. +nl_after_brace_close = false # true/false + +# Whether to add a newline after a virtual brace close, +# as in 'if (foo) a++; return;'. +nl_after_vbrace_close = false # true/false + +# Add or remove newline between the close brace and identifier, +# as in 'struct { int a; } b;'. Affects enumerations, unions and +# structures. If set to ignore, uses nl_after_brace_close. +nl_brace_struct_var = ignore # ignore/add/remove/force + +# Whether to alter newlines in '#define' macros. +nl_define_macro = false # true/false + +# Whether to alter newlines between consecutive parenthesis closes. The number +# of closing parentheses in a line will depend on respective open parenthesis +# lines. +nl_squeeze_paren_close = false # true/false + +# Whether to remove blanks after '#ifxx' and '#elxx', or before '#elxx' and +# '#endif'. Does not affect top-level #ifdefs. +nl_squeeze_ifdef = false # true/false + +# Makes the nl_squeeze_ifdef option affect the top-level #ifdefs as well. +nl_squeeze_ifdef_top_level = false # true/false + +# Add or remove blank line before 'if'. +nl_before_if = ignore # ignore/add/remove/force + +# Add or remove blank line after 'if' statement. Add/Force work only if the +# next token is not a closing brace. +nl_after_if = ignore # ignore/add/remove/force + +# Add or remove blank line before 'for'. +nl_before_for = ignore # ignore/add/remove/force + +# Add or remove blank line after 'for' statement. +nl_after_for = ignore # ignore/add/remove/force + +# Add or remove blank line before 'while'. +nl_before_while = ignore # ignore/add/remove/force + +# Add or remove blank line after 'while' statement. +nl_after_while = ignore # ignore/add/remove/force + +# Add or remove blank line before 'switch'. +nl_before_switch = ignore # ignore/add/remove/force + +# Add or remove blank line after 'switch' statement. +nl_after_switch = ignore # ignore/add/remove/force + +# Add or remove blank line before 'synchronized'. +nl_before_synchronized = ignore # ignore/add/remove/force + +# Add or remove blank line after 'synchronized' statement. +nl_after_synchronized = ignore # ignore/add/remove/force + +# Add or remove blank line before 'do'. +nl_before_do = ignore # ignore/add/remove/force + +# Add or remove blank line after 'do/while' statement. +nl_after_do = ignore # ignore/add/remove/force + +# Whether to put a blank line before 'return' statements, unless after an open +# brace. +nl_before_return = false # true/false + +# Whether to put a blank line after 'return' statements, unless followed by a +# close brace. +nl_after_return = false # true/false + +# (Java) Whether to put a blank line before a member '.' or '->' operators. +nl_before_member = ignore # ignore/add/remove/force + +# (Java) Whether to put a blank line after a member '.' or '->' operators. +nl_after_member = ignore # ignore/add/remove/force + +# Whether to double-space commented-entries in 'struct'/'union'/'enum'. +nl_ds_struct_enum_cmt = false # true/false + +# Whether to force a newline before '}' of a 'struct'/'union'/'enum'. +# (Lower priority than eat_blanks_before_close_brace.) +nl_ds_struct_enum_close_brace = false # true/false + +# Add or remove newline before or after (depending on pos_class_colon) a class +# colon, as in 'class Foo : public Bar'. +nl_class_colon = ignore # ignore/add/remove/force + +# Add or remove newline around a class constructor colon. The exact position +# depends on nl_constr_init_args, pos_constr_colon and pos_constr_comma. +nl_constr_colon = ignore # ignore/add/remove/force + +# Whether to collapse a two-line namespace, like 'namespace foo\n{ decl; }' +# into a single line. If true, prevents other brace newline rules from turning +# such code into four lines. +nl_namespace_two_to_one_liner = false # true/false + +# Whether to remove a newline in simple unbraced if statements, turning them +# into one-liners, as in 'if(b)\n i++;' => 'if(b) i++;'. +nl_create_if_one_liner = false # true/false + +# Whether to remove a newline in simple unbraced for statements, turning them +# into one-liners, as in 'for (...)\n stmt;' => 'for (...) stmt;'. +nl_create_for_one_liner = false # true/false + +# Whether to remove a newline in simple unbraced while statements, turning +# them into one-liners, as in 'while (expr)\n stmt;' => 'while (expr) stmt;'. +nl_create_while_one_liner = false # true/false + +# Whether to collapse a function definition whose body (not counting braces) +# is only one line so that the entire definition (prototype, braces, body) is +# a single line. +nl_create_func_def_one_liner = false # true/false + +# Whether to collapse a function definition whose body (not counting braces) +# is only one line so that the entire definition (prototype, braces, body) is +# a single line. +nl_create_list_one_liner = false # true/false + +# Whether to split one-line simple unbraced if statements into two lines by +# adding a newline, as in 'if(b) i++;'. +nl_split_if_one_liner = false # true/false + +# Whether to split one-line simple unbraced for statements into two lines by +# adding a newline, as in 'for (...) stmt;'. +nl_split_for_one_liner = false # true/false + +# Whether to split one-line simple unbraced while statements into two lines by +# adding a newline, as in 'while (expr) stmt;'. +nl_split_while_one_liner = false # true/false + +# +# Blank line options +# + +# The maximum number of consecutive newlines (3 = 2 blank lines). +nl_max = 0 # unsigned number + +# The maximum number of consecutive newlines in a function. +nl_max_blank_in_func = 0 # unsigned number + +# The number of newlines before a function prototype. +nl_before_func_body_proto = 0 # unsigned number + +# The number of newlines before a multi-line function definition. +nl_before_func_body_def = 0 # unsigned number + +# The number of newlines before a class constructor/destructor prototype. +nl_before_func_class_proto = 0 # unsigned number + +# The number of newlines before a class constructor/destructor definition. +nl_before_func_class_def = 0 # unsigned number + +# The number of newlines after a function prototype. +nl_after_func_proto = 0 # unsigned number + +# The number of newlines after a function prototype, if not followed by +# another function prototype. +nl_after_func_proto_group = 0 # unsigned number + +# The number of newlines after a class constructor/destructor prototype. +nl_after_func_class_proto = 0 # unsigned number + +# The number of newlines after a class constructor/destructor prototype, +# if not followed by another constructor/destructor prototype. +nl_after_func_class_proto_group = 0 # unsigned number + +# Whether one-line method definitions inside a class body should be treated +# as if they were prototypes for the purposes of adding newlines. +# +# Requires nl_class_leave_one_liners=true. Overrides nl_before_func_body_def +# and nl_before_func_class_def for one-liners. +nl_class_leave_one_liner_groups = false # true/false + +# The number of newlines after '}' of a multi-line function body. +nl_after_func_body = 0 # unsigned number + +# The number of newlines after '}' of a multi-line function body in a class +# declaration. Also affects class constructors/destructors. +# +# Overrides nl_after_func_body. +nl_after_func_body_class = 0 # unsigned number + +# The number of newlines after '}' of a single line function body. Also +# affects class constructors/destructors. +# +# Overrides nl_after_func_body and nl_after_func_body_class. +nl_after_func_body_one_liner = 0 # unsigned number + +# The number of blank lines after a block of variable definitions at the top +# of a function body. +# +# 0: No change (default). +nl_func_var_def_blk = 0 # unsigned number + +# The number of newlines before a block of typedefs. If nl_after_access_spec +# is non-zero, that option takes precedence. +# +# 0: No change (default). +nl_typedef_blk_start = 0 # unsigned number + +# The number of newlines after a block of typedefs. +# +# 0: No change (default). +nl_typedef_blk_end = 0 # unsigned number + +# The maximum number of consecutive newlines within a block of typedefs. +# +# 0: No change (default). +nl_typedef_blk_in = 0 # unsigned number + +# The number of newlines before a block of variable definitions not at the top +# of a function body. If nl_after_access_spec is non-zero, that option takes +# precedence. +# +# 0: No change (default). +nl_var_def_blk_start = 0 # unsigned number + +# The number of newlines after a block of variable definitions not at the top +# of a function body. +# +# 0: No change (default). +nl_var_def_blk_end = 0 # unsigned number + +# The maximum number of consecutive newlines within a block of variable +# definitions. +# +# 0: No change (default). +nl_var_def_blk_in = 0 # unsigned number + +# The minimum number of newlines before a multi-line comment. +# Doesn't apply if after a brace open or another multi-line comment. +nl_before_block_comment = 0 # unsigned number + +# The minimum number of newlines before a single-line C comment. +# Doesn't apply if after a brace open or other single-line C comments. +nl_before_c_comment = 0 # unsigned number + +# The minimum number of newlines before a CPP comment. +# Doesn't apply if after a brace open or other CPP comments. +nl_before_cpp_comment = 0 # unsigned number + +# Whether to force a newline after a multi-line comment. +nl_after_multiline_comment = false # true/false + +# Whether to force a newline after a label's colon. +nl_after_label_colon = false # true/false + +# The number of newlines after '}' or ';' of a struct/enum/union definition. +nl_after_struct = 0 # unsigned number + +# The number of newlines before a class definition. +nl_before_class = 0 # unsigned number + +# The number of newlines after '}' or ';' of a class definition. +nl_after_class = 0 # unsigned number + +# The number of newlines before a namespace. +nl_before_namespace = 0 # unsigned number + +# The number of newlines after '{' of a namespace. This also adds newlines +# before the matching '}'. +# +# 0: Apply eat_blanks_after_open_brace or eat_blanks_before_close_brace if +# applicable, otherwise no change. +# +# Overrides eat_blanks_after_open_brace and eat_blanks_before_close_brace. +nl_inside_namespace = 0 # unsigned number + +# The number of newlines after '}' of a namespace. +nl_after_namespace = 0 # unsigned number + +# The number of newlines before an access specifier label. This also includes +# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count +# if after a brace open. +# +# 0: No change (default). +nl_before_access_spec = 0 # unsigned number + +# The number of newlines after an access specifier label. This also includes +# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count +# if after a brace open. +# +# 0: No change (default). +# +# Overrides nl_typedef_blk_start and nl_var_def_blk_start. +nl_after_access_spec = 0 # unsigned number + +# The number of newlines between a function definition and the function +# comment, as in '// comment\n void foo() {...}'. +# +# 0: No change (default). +nl_comment_func_def = 0 # unsigned number + +# The number of newlines after a try-catch-finally block that isn't followed +# by a brace close. +# +# 0: No change (default). +nl_after_try_catch_finally = 0 # unsigned number + +# (C#) The number of newlines before and after a property, indexer or event +# declaration. +# +# 0: No change (default). +nl_around_cs_property = 0 # unsigned number + +# (C#) The number of newlines between the get/set/add/remove handlers. +# +# 0: No change (default). +nl_between_get_set = 0 # unsigned number + +# (C#) Add or remove newline between property and the '{'. +nl_property_brace = ignore # ignore/add/remove/force + +# Whether to remove blank lines after '{'. +eat_blanks_after_open_brace = false # true/false + +# Whether to remove blank lines before '}'. +eat_blanks_before_close_brace = false # true/false + +# How aggressively to remove extra newlines not in preprocessor. +# +# 0: No change (default) +# 1: Remove most newlines not handled by other config +# 2: Remove all newlines and reformat completely by config +nl_remove_extra_newlines = 0 # unsigned number + +# (Java) Add or remove newline after an annotation statement. Only affects +# annotations that are after a newline. +nl_after_annotation = ignore # ignore/add/remove/force + +# (Java) Add or remove newline between two annotations. +nl_between_annotation = ignore # ignore/add/remove/force + +# +# Positioning options +# + +# The position of arithmetic operators in wrapped expressions. +pos_arith = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of assignment in wrapped expressions. Do not affect '=' +# followed by '{'. +pos_assign = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of Boolean operators in wrapped expressions. +pos_bool = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of comparison operators in wrapped expressions. +pos_compare = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of conditional operators, as in the '?' and ':' of +# 'expr ? stmt : stmt', in wrapped expressions. +pos_conditional = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of the comma in wrapped expressions. +pos_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of the comma in enum entries. +pos_enum_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of the comma in the base class list if there is more than one +# line. Affects nl_class_init_args. +pos_class_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of the comma in the constructor initialization list. +# Related to nl_constr_colon, nl_constr_init_args and pos_constr_colon. +pos_constr_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of trailing/leading class colon, between class and base class +# list. Affects nl_class_colon. +pos_class_colon = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# The position of colons between constructor and member initialization. +# Related to nl_constr_colon, nl_constr_init_args and pos_constr_comma. +pos_constr_colon = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force + +# +# Line splitting options +# + +# Try to limit code width to N columns. +code_width = 0 # unsigned number + +# Whether to fully split long 'for' statements at semi-colons. +ls_for_split_full = false # true/false + +# Whether to fully split long function prototypes/calls at commas. +# The option ls_code_width has priority over the option ls_func_split_full. +ls_func_split_full = false # true/false + +# Whether to split lines as close to code_width as possible and ignore some +# groupings. +# The option ls_code_width has priority over the option ls_func_split_full. +ls_code_width = false # true/false + +# +# Code alignment options (not left column spaces/tabs) +# + +# Whether to keep non-indenting tabs. +align_keep_tabs = false # true/false + +# Whether to use tabs for aligning. +align_with_tabs = false # true/false + +# Whether to bump out to the next tab when aligning. +align_on_tabstop = false # true/false + +# Whether to right-align numbers. +align_number_right = false # true/false + +# Whether to keep whitespace not required for alignment. +align_keep_extra_space = false # true/false + +# Whether to align variable definitions in prototypes and functions. +align_func_params = false # true/false + +# The span for aligning parameter definitions in function on parameter name. +# +# 0: Don't align (default). +align_func_params_span = 0 # unsigned number + +# The threshold for aligning function parameter definitions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_func_params_thresh = 0 # number + +# The gap for aligning function parameter definitions. +align_func_params_gap = 0 # unsigned number + +# The span for aligning constructor value. +# +# 0: Don't align (default). +align_constr_value_span = 0 # unsigned number + +# The threshold for aligning constructor value. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_constr_value_thresh = 0 # number + +# The gap for aligning constructor value. +align_constr_value_gap = 0 # unsigned number + +# Whether to align parameters in single-line functions that have the same +# name. The function names must already be aligned with each other. +align_same_func_call_params = false # true/false + +# The span for aligning function-call parameters for single line functions. +# +# 0: Don't align (default). +align_same_func_call_params_span = 0 # unsigned number + +# The threshold for aligning function-call parameters for single line +# functions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_same_func_call_params_thresh = 0 # number + +# The span for aligning variable definitions. +# +# 0: Don't align (default). +align_var_def_span = 0 # unsigned number + +# How to consider (or treat) the '*' in the alignment of variable definitions. +# +# 0: Part of the type 'void * foo;' (default) +# 1: Part of the variable 'void *foo;' +# 2: Dangling 'void *foo;' +# Dangling: the '*' will not be taken into account when aligning. +align_var_def_star_style = 0 # unsigned number + +# How to consider (or treat) the '&' in the alignment of variable definitions. +# +# 0: Part of the type 'long & foo;' (default) +# 1: Part of the variable 'long &foo;' +# 2: Dangling 'long &foo;' +# Dangling: the '&' will not be taken into account when aligning. +align_var_def_amp_style = 0 # unsigned number + +# The threshold for aligning variable definitions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_var_def_thresh = 0 # number + +# The gap for aligning variable definitions. +align_var_def_gap = 0 # unsigned number + +# Whether to align the colon in struct bit fields. +align_var_def_colon = false # true/false + +# The gap for aligning the colon in struct bit fields. +align_var_def_colon_gap = 0 # unsigned number + +# Whether to align any attribute after the variable name. +align_var_def_attribute = false # true/false + +# Whether to align inline struct/enum/union variable definitions. +align_var_def_inline = false # true/false + +# The span for aligning on '=' in assignments. +# +# 0: Don't align (default). +align_assign_span = 0 # unsigned number + +# The span for aligning on '=' in function prototype modifier. +# +# 0: Don't align (default). +align_assign_func_proto_span = 0 # unsigned number + +# The threshold for aligning on '=' in assignments. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_assign_thresh = 0 # number + +# How to apply align_assign_span to function declaration "assignments", i.e. +# 'virtual void foo() = 0' or '~foo() = {default|delete}'. +# +# 0: Align with other assignments (default) +# 1: Align with each other, ignoring regular assignments +# 2: Don't align +align_assign_decl_func = 0 # unsigned number + +# The span for aligning on '=' in enums. +# +# 0: Don't align (default). +align_enum_equ_span = 0 # unsigned number + +# The threshold for aligning on '=' in enums. +# Use a negative number for absolute thresholds. +# +# 0: no limit (default). +align_enum_equ_thresh = 0 # number + +# The span for aligning class member definitions. +# +# 0: Don't align (default). +align_var_class_span = 0 # unsigned number + +# The threshold for aligning class member definitions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_var_class_thresh = 0 # number + +# The gap for aligning class member definitions. +align_var_class_gap = 0 # unsigned number + +# The span for aligning struct/union member definitions. +# +# 0: Don't align (default). +align_var_struct_span = 0 # unsigned number + +# The threshold for aligning struct/union member definitions. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_var_struct_thresh = 0 # number + +# The gap for aligning struct/union member definitions. +align_var_struct_gap = 0 # unsigned number + +# The span for aligning struct initializer values. +# +# 0: Don't align (default). +align_struct_init_span = 0 # unsigned number + +# The span for aligning single-line typedefs. +# +# 0: Don't align (default). +align_typedef_span = 0 # unsigned number + +# The minimum space between the type and the synonym of a typedef. +align_typedef_gap = 0 # unsigned number + +# How to align typedef'd functions with other typedefs. +# +# 0: Don't mix them at all (default) +# 1: Align the open parenthesis with the types +# 2: Align the function type name with the other type names +align_typedef_func = 0 # unsigned number + +# How to consider (or treat) the '*' in the alignment of typedefs. +# +# 0: Part of the typedef type, 'typedef int * pint;' (default) +# 1: Part of type name: 'typedef int *pint;' +# 2: Dangling: 'typedef int *pint;' +# Dangling: the '*' will not be taken into account when aligning. +align_typedef_star_style = 0 # unsigned number + +# How to consider (or treat) the '&' in the alignment of typedefs. +# +# 0: Part of the typedef type, 'typedef int & intref;' (default) +# 1: Part of type name: 'typedef int &intref;' +# 2: Dangling: 'typedef int &intref;' +# Dangling: the '&' will not be taken into account when aligning. +align_typedef_amp_style = 0 # unsigned number + +# The span for aligning comments that end lines. +# +# 0: Don't align (default). +align_right_cmt_span = 0 # unsigned number + +# Minimum number of columns between preceding text and a trailing comment in +# order for the comment to qualify for being aligned. Must be non-zero to have +# an effect. +align_right_cmt_gap = 0 # unsigned number + +# If aligning comments, whether to mix with comments after '}' and #endif with +# less than three spaces before the comment. +align_right_cmt_mix = false # true/false + +# Whether to only align trailing comments that are at the same brace level. +align_right_cmt_same_level = false # true/false + +# Minimum column at which to align trailing comments. Comments which are +# aligned beyond this column, but which can be aligned in a lesser column, +# may be "pulled in". +# +# 0: Ignore (default). +align_right_cmt_at_col = 0 # unsigned number + +# The span for aligning function prototypes. +# +# 0: Don't align (default). +align_func_proto_span = 0 # unsigned number + +# The threshold for aligning function prototypes. +# Use a negative number for absolute thresholds. +# +# 0: No limit (default). +align_func_proto_thresh = 0 # number + +# Minimum gap between the return type and the function name. +align_func_proto_gap = 0 # unsigned number + +# Whether to align function prototypes on the 'operator' keyword instead of +# what follows. +align_on_operator = false # true/false + +# Whether to mix aligning prototype and variable declarations. If true, +# align_var_def_XXX options are used instead of align_func_proto_XXX options. +align_mix_var_proto = false # true/false + +# Whether to align single-line functions with function prototypes. +# Uses align_func_proto_span. +align_single_line_func = false # true/false + +# Whether to align the open brace of single-line functions. +# Requires align_single_line_func=true. Uses align_func_proto_span. +align_single_line_brace = false # true/false + +# Gap for align_single_line_brace. +align_single_line_brace_gap = 0 # unsigned number + +# (OC) The span for aligning Objective-C message specifications. +# +# 0: Don't align (default). +align_oc_msg_spec_span = 0 # unsigned number + +# Whether to align macros wrapped with a backslash and a newline. This will +# not work right if the macro contains a multi-line comment. +align_nl_cont = false # true/false + +# Whether to align macro functions and variables together. +align_pp_define_together = false # true/false + +# The span for aligning on '#define' bodies. +# +# =0: Don't align (default) +# >0: Number of lines (including comments) between blocks +align_pp_define_span = 0 # unsigned number + +# The minimum space between label and value of a preprocessor define. +align_pp_define_gap = 0 # unsigned number + +# Whether to align lines that start with '<<' with previous '<<'. +# +# Default: true +align_left_shift = true # true/false + +# Whether to align text after 'asm volatile ()' colons. +align_asm_colon = false # true/false + +# (OC) Span for aligning parameters in an Objective-C message call +# on the ':'. +# +# 0: Don't align. +align_oc_msg_colon_span = 0 # unsigned number + +# (OC) Whether to always align with the first parameter, even if it is too +# short. +align_oc_msg_colon_first = false # true/false + +# (OC) Whether to align parameters in an Objective-C '+' or '-' declaration +# on the ':'. +align_oc_decl_colon = false # true/false + +# +# Comment modification options +# + +# Try to wrap comments at N columns. +cmt_width = 0 # unsigned number + +# How to reflow comments. +# +# 0: No reflowing (apart from the line wrapping due to cmt_width) (default) +# 1: No touching at all +# 2: Full reflow +cmt_reflow_mode = 0 # unsigned number + +# Whether to convert all tabs to spaces in comments. If false, tabs in +# comments are left alone, unless used for indenting. +cmt_convert_tab_to_spaces = false # true/false + +# Whether to apply changes to multi-line comments, including cmt_width, +# keyword substitution and leading chars. +# +# Default: true +cmt_indent_multi = true # true/false + +# Whether to group c-comments that look like they are in a block. +cmt_c_group = false # true/false + +# Whether to put an empty '/*' on the first line of the combined c-comment. +cmt_c_nl_start = false # true/false + +# Whether to add a newline before the closing '*/' of the combined c-comment. +cmt_c_nl_end = false # true/false + +# Whether to change cpp-comments into c-comments. +cmt_cpp_to_c = false # true/false + +# Whether to group cpp-comments that look like they are in a block. Only +# meaningful if cmt_cpp_to_c=true. +cmt_cpp_group = false # true/false + +# Whether to put an empty '/*' on the first line of the combined cpp-comment +# when converting to a c-comment. +# +# Requires cmt_cpp_to_c=true and cmt_cpp_group=true. +cmt_cpp_nl_start = false # true/false + +# Whether to add a newline before the closing '*/' of the combined cpp-comment +# when converting to a c-comment. +# +# Requires cmt_cpp_to_c=true and cmt_cpp_group=true. +cmt_cpp_nl_end = false # true/false + +# Whether to put a star on subsequent comment lines. +cmt_star_cont = false # true/false + +# The number of spaces to insert at the start of subsequent comment lines. +cmt_sp_before_star_cont = 0 # unsigned number + +# The number of spaces to insert after the star on subsequent comment lines. +cmt_sp_after_star_cont = 0 # unsigned number + +# For multi-line comments with a '*' lead, remove leading spaces if the first +# and last lines of the comment are the same length. +# +# Default: true +cmt_multi_check_last = true # true/false + +# For multi-line comments with a '*' lead, remove leading spaces if the first +# and last lines of the comment are the same length AND if the length is +# bigger as the first_len minimum. +# +# Default: 4 +cmt_multi_first_len_minimum = 4 # unsigned number + +# Path to a file that contains text to insert at the beginning of a file if +# the file doesn't start with a C/C++ comment. If the inserted text contains +# '$(filename)', that will be replaced with the current file's name. +cmt_insert_file_header = "" # string + +# Path to a file that contains text to insert at the end of a file if the +# file doesn't end with a C/C++ comment. If the inserted text contains +# '$(filename)', that will be replaced with the current file's name. +cmt_insert_file_footer = "" # string + +# Path to a file that contains text to insert before a function definition if +# the function isn't preceded by a C/C++ comment. If the inserted text +# contains '$(function)', '$(javaparam)' or '$(fclass)', these will be +# replaced with, respectively, the name of the function, the javadoc '@param' +# and '@return' stuff, or the name of the class to which the member function +# belongs. +cmt_insert_func_header = "" # string + +# Path to a file that contains text to insert before a class if the class +# isn't preceded by a C/C++ comment. If the inserted text contains '$(class)', +# that will be replaced with the class name. +cmt_insert_class_header = "" # string + +# Path to a file that contains text to insert before an Objective-C message +# specification, if the method isn't preceded by a C/C++ comment. If the +# inserted text contains '$(message)' or '$(javaparam)', these will be +# replaced with, respectively, the name of the function, or the javadoc +# '@param' and '@return' stuff. +cmt_insert_oc_msg_header = "" # string + +# Whether a comment should be inserted if a preprocessor is encountered when +# stepping backwards from a function name. +# +# Applies to cmt_insert_oc_msg_header, cmt_insert_func_header and +# cmt_insert_class_header. +cmt_insert_before_preproc = false # true/false + +# Whether a comment should be inserted if a function is declared inline to a +# class definition. +# +# Applies to cmt_insert_func_header. +# +# Default: true +cmt_insert_before_inlines = true # true/false + +# Whether a comment should be inserted if the function is a class constructor +# or destructor. +# +# Applies to cmt_insert_func_header. +cmt_insert_before_ctor_dtor = false # true/false + +# +# Code modifying options (non-whitespace) +# + +# Add or remove braces on a single-line 'do' statement. +mod_full_brace_do = ignore # ignore/add/remove/force + +# Add or remove braces on a single-line 'for' statement. +mod_full_brace_for = ignore # ignore/add/remove/force + +# (Pawn) Add or remove braces on a single-line function definition. +mod_full_brace_function = ignore # ignore/add/remove/force + +# Add or remove braces on a single-line 'if' statement. Braces will not be +# removed if the braced statement contains an 'else'. +mod_full_brace_if = ignore # ignore/add/remove/force + +# Whether to enforce that all blocks of an 'if'/'else if'/'else' chain either +# have, or do not have, braces. If true, braces will be added if any block +# needs braces, and will only be removed if they can be removed from all +# blocks. +# +# Overrides mod_full_brace_if. +mod_full_brace_if_chain = false # true/false + +# Whether to add braces to all blocks of an 'if'/'else if'/'else' chain. +# If true, mod_full_brace_if_chain will only remove braces from an 'if' that +# does not have an 'else if' or 'else'. +mod_full_brace_if_chain_only = false # true/false + +# Add or remove braces on single-line 'while' statement. +mod_full_brace_while = ignore # ignore/add/remove/force + +# Add or remove braces on single-line 'using ()' statement. +mod_full_brace_using = ignore # ignore/add/remove/force + +# Don't remove braces around statements that span N newlines +mod_full_brace_nl = 0 # unsigned number + +# Whether to prevent removal of braces from 'if'/'for'/'while'/etc. blocks +# which span multiple lines. +# +# Affects: +# mod_full_brace_for +# mod_full_brace_if +# mod_full_brace_if_chain +# mod_full_brace_if_chain_only +# mod_full_brace_while +# mod_full_brace_using +# +# Does not affect: +# mod_full_brace_do +# mod_full_brace_function +mod_full_brace_nl_block_rem_mlcond = false # true/false + +# Add or remove unnecessary parenthesis on 'return' statement. +mod_paren_on_return = ignore # ignore/add/remove/force + +# (Pawn) Whether to change optional semicolons to real semicolons. +mod_pawn_semicolon = false # true/false + +# Whether to fully parenthesize Boolean expressions in 'while' and 'if' +# statement, as in 'if (a && b > c)' => 'if (a && (b > c))'. +mod_full_paren_if_bool = false # true/false + +# Whether to remove superfluous semicolons. +mod_remove_extra_semicolon = false # true/false + +# If a function body exceeds the specified number of newlines and doesn't have +# a comment after the close brace, a comment will be added. +mod_add_long_function_closebrace_comment = 0 # unsigned number + +# If a namespace body exceeds the specified number of newlines and doesn't +# have a comment after the close brace, a comment will be added. +mod_add_long_namespace_closebrace_comment = 0 # unsigned number + +# If a class body exceeds the specified number of newlines and doesn't have a +# comment after the close brace, a comment will be added. +mod_add_long_class_closebrace_comment = 0 # unsigned number + +# If a switch body exceeds the specified number of newlines and doesn't have a +# comment after the close brace, a comment will be added. +mod_add_long_switch_closebrace_comment = 0 # unsigned number + +# If an #ifdef body exceeds the specified number of newlines and doesn't have +# a comment after the #endif, a comment will be added. +mod_add_long_ifdef_endif_comment = 0 # unsigned number + +# If an #ifdef or #else body exceeds the specified number of newlines and +# doesn't have a comment after the #else, a comment will be added. +mod_add_long_ifdef_else_comment = 0 # unsigned number + +# Whether to take care of the case by the mod_sort_xx options. +mod_sort_case_sensitive = false # true/false + +# Whether to sort consecutive single-line 'import' statements. +mod_sort_import = false # true/false + +# (C#) Whether to sort consecutive single-line 'using' statements. +mod_sort_using = false # true/false + +# Whether to sort consecutive single-line '#include' statements (C/C++) and +# '#import' statements (Objective-C). Be aware that this has the potential to +# break your code if your includes/imports have ordering dependencies. +mod_sort_include = false # true/false + +# Whether to move a 'break' that appears after a fully braced 'case' before +# the close brace, as in 'case X: { ... } break;' => 'case X: { ... break; }'. +mod_move_case_break = false # true/false + +# Add or remove braces around a fully braced case statement. Will only remove +# braces if there are no variable declarations in the block. +mod_case_brace = ignore # ignore/add/remove/force + +# Whether to remove a void 'return;' that appears as the last statement in a +# function. +mod_remove_empty_return = false # true/false + +# Add or remove the comma after the last value of an enumeration. +mod_enum_last_comma = ignore # ignore/add/remove/force + +# (OC) Whether to organize the properties. If true, properties will be +# rearranged according to the mod_sort_oc_property_*_weight factors. +mod_sort_oc_properties = false # true/false + +# (OC) Weight of a class property modifier. +mod_sort_oc_property_class_weight = 0 # number + +# (OC) Weight of 'atomic' and 'nonatomic'. +mod_sort_oc_property_thread_safe_weight = 0 # number + +# (OC) Weight of 'readwrite' when organizing properties. +mod_sort_oc_property_readwrite_weight = 0 # number + +# (OC) Weight of a reference type specifier ('retain', 'copy', 'assign', +# 'weak', 'strong') when organizing properties. +mod_sort_oc_property_reference_weight = 0 # number + +# (OC) Weight of getter type ('getter=') when organizing properties. +mod_sort_oc_property_getter_weight = 0 # number + +# (OC) Weight of setter type ('setter=') when organizing properties. +mod_sort_oc_property_setter_weight = 0 # number + +# (OC) Weight of nullability type ('nullable', 'nonnull', 'null_unspecified', +# 'null_resettable') when organizing properties. +mod_sort_oc_property_nullability_weight = 0 # number + +# +# Preprocessor options +# + +# Add or remove indentation of preprocessor directives inside #if blocks +# at brace level 0 (file-level). +pp_indent = ignore # ignore/add/remove/force + +# Whether to indent #if/#else/#endif at the brace level. If false, these are +# indented from column 1. +pp_indent_at_level = false # true/false + +# Specifies the number of columns to indent preprocessors per level +# at brace level 0 (file-level). If pp_indent_at_level=false, also specifies +# the number of columns to indent preprocessors per level +# at brace level > 0 (function-level). +# +# Default: 1 +pp_indent_count = 1 # unsigned number + +# Add or remove space after # based on pp_level of #if blocks. +pp_space = ignore # ignore/add/remove/force + +# Sets the number of spaces per level added with pp_space. +pp_space_count = 0 # unsigned number + +# The indent for '#region' and '#endregion' in C# and '#pragma region' in +# C/C++. Negative values decrease indent down to the first column. +pp_indent_region = 0 # number + +# Whether to indent the code between #region and #endregion. +pp_region_indent_code = false # true/false + +# If pp_indent_at_level=true, sets the indent for #if, #else and #endif when +# not at file-level. Negative values decrease indent down to the first column. +# +# =0: Indent preprocessors using output_tab_size +# >0: Column at which all preprocessors will be indented +pp_indent_if = 0 # number + +# Whether to indent the code between #if, #else and #endif. +pp_if_indent_code = false # true/false + +# Whether to indent '#define' at the brace level. If false, these are +# indented from column 1. +pp_define_at_level = false # true/false + +# Whether to ignore the '#define' body while formatting. +pp_ignore_define_body = false # true/false + +# Whether to indent case statements between #if, #else, and #endif. +# Only applies to the indent of the preprocesser that the case statements +# directly inside of. +# +# Default: true +pp_indent_case = true # true/false + +# Whether to indent whole function definitions between #if, #else, and #endif. +# Only applies to the indent of the preprocesser that the function definition +# is directly inside of. +# +# Default: true +pp_indent_func_def = true # true/false + +# Whether to indent extern C blocks between #if, #else, and #endif. +# Only applies to the indent of the preprocesser that the extern block is +# directly inside of. +# +# Default: true +pp_indent_extern = true # true/false + +# Whether to indent braces directly inside #if, #else, and #endif. +# Only applies to the indent of the preprocesser that the braces are directly +# inside of. +# +# Default: true +pp_indent_brace = true # true/false + +# +# Sort includes options +# + +# The regex for include category with priority 0. +include_category_0 = "" # string + +# The regex for include category with priority 1. +include_category_1 = "" # string + +# The regex for include category with priority 2. +include_category_2 = "" # string + +# +# Use or Do not Use options +# + +# true: indent_func_call_param will be used (default) +# false: indent_func_call_param will NOT be used +# +# Default: true +use_indent_func_call_param = true # true/false + +# The value of the indentation for a continuation line is calculated +# differently if the statement is: +# - a declaration: your case with QString fileName ... +# - an assignment: your case with pSettings = new QSettings( ... +# +# At the second case the indentation value might be used twice: +# - at the assignment +# - at the function call (if present) +# +# To prevent the double use of the indentation value, use this option with the +# value 'true'. +# +# true: indent_continue will be used only once +# false: indent_continue will be used every time (default) +use_indent_continue_only_once = false # true/false + +# The value might be used twice: +# - at the assignment +# - at the opening brace +# +# To prevent the double use of the indentation value, use this option with the +# value 'true'. +# +# true: indentation will be used only once +# false: indentation will be used every time (default) +indent_cpp_lambda_only_once = false # true/false + +# Whether sp_after_angle takes precedence over sp_inside_fparen. This was the +# historic behavior, but is probably not the desired behavior, so this is off +# by default. +use_sp_after_angle_always = false # true/false + +# Whether to apply special formatting for Qt SIGNAL/SLOT macros. Essentially, +# this tries to format these so that they match Qt's normalized form (i.e. the +# result of QMetaObject::normalizedSignature), which can slightly improve the +# performance of the QObject::connect call, rather than how they would +# otherwise be formatted. +# +# See options_for_QT.cpp for details. +# +# Default: true +use_options_overriding_for_qt_macros = true # true/false + +# +# Warn levels - 1: error, 2: warning (default), 3: note +# + +# (C#) Warning is given if doing tab-to-\t replacement and we have found one +# in a C# verbatim string literal. +# +# Default: 2 +warn_level_tabs_found_in_verbatim_string_literals = 2 # unsigned number + +# Meaning of the settings: +# Ignore - do not do any changes +# Add - makes sure there is 1 or more space/brace/newline/etc +# Force - makes sure there is exactly 1 space/brace/newline/etc, +# behaves like Add in some contexts +# Remove - removes space/brace/newline/etc +# +# +# - Token(s) can be treated as specific type(s) with the 'set' option: +# `set tokenType tokenString [tokenString...]` +# +# Example: +# `set BOOL __AND__ __OR__` +# +# tokenTypes are defined in src/token_enum.h, use them without the +# 'CT_' prefix: 'CT_BOOL' => 'BOOL' +# +# +# - Token(s) can be treated as type(s) with the 'type' option. +# `type tokenString [tokenString...]` +# +# Example: +# `type int c_uint_8 Rectangle` +# +# This can also be achieved with `set TYPE int c_uint_8 Rectangle` +# +# +# To embed whitespace in tokenStrings use the '\' escape character, or quote +# the tokenStrings. These quotes are supported: "'` +# +# +# - Support for the auto detection of languages through the file ending can be +# added using the 'file_ext' command. +# `file_ext langType langString [langString..]` +# +# Example: +# `file_ext CPP .ch .cxx .cpp.in` +# +# langTypes are defined in uncrusify_types.h in the lang_flag_e enum, use +# them without the 'LANG_' prefix: 'LANG_CPP' => 'CPP' +# +# +# - Custom macro-based indentation can be set up using 'macro-open', +# 'macro-else' and 'macro-close'. +# `(macro-open | macro-else | macro-close) tokenString` +# +# Example: +# `macro-open BEGIN_TEMPLATE_MESSAGE_MAP` +# `macro-open BEGIN_MESSAGE_MAP` +# `macro-close END_MESSAGE_MAP` +# +# +# option(s) with 'not default' value: 0 +# + +file_ext CPP .ch .cxx .cpp.in .cc .hh + diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 98018ca0e..437a761c4 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -29,38 +29,38 @@ int main() ErrorInfo { .level = lvlInfo, .name = "Info name", .description = "Info description", - }); + }); logger->logEI( ErrorInfo { .level = lvlTalkative, .name = "Talkative name", .description = "Talkative description", - }); + }); logger->logEI( ErrorInfo { .level = lvlChatty, .name = "Chatty name", .description = "Chatty description", - }); + }); logger->logEI( ErrorInfo { .level = lvlDebug, .name = "Debug name", .description = "Debug description", - }); + }); logger->logEI( ErrorInfo { .level = lvlVomit, .name = "Vomit name", .description = "Vomit description", - }); + }); // Error in a program; no hint and no nix code. logError( ErrorInfo { .name = "name", .description = "error description", - }); + }); // Warning with name, description, and hint. // The hintfmt function makes all the substituted text yellow. @@ -68,7 +68,7 @@ int main() ErrorInfo { .name = "name", .description = "error description", .hint = hintfmt("there was a %1%", "warning"), - }); + }); // Warning with nix file, line number, column, and the lines of @@ -80,28 +80,28 @@ int main() ErrorInfo { .name = "warning name", .description = "warning description", .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), + "yellow", + "values"), .nixCode = NixCode { .errPos = Pos(problem_file, 40, 13), .prevLineOfCode = std::nullopt, .errLineOfCode = "this is the problem line of code", .nextLineOfCode = std::nullopt - }}); + }}); // Error with previous and next lines of code. logError( ErrorInfo { .name = "error name", .description = "error description", .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), + "yellow", + "values"), .nixCode = NixCode { .errPos = Pos(problem_file, 40, 13), .prevLineOfCode = std::optional("previous line of code"), .errLineOfCode = "this is the problem line of code", .nextLineOfCode = std::optional("next line of code"), - }}); + }}); return 0; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 79e3f90e5..a8bae0a61 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -234,7 +234,8 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); }; + throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); + }; void setName(Symbol & name); string showNamePos() const; COMMON_METHODS diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8f0e0aa1a..91fa1ccd8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -45,17 +45,17 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) // previous line of code. if (nixCode.prevLineOfCode.has_value()) { std::cout << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line - 1), - *nixCode.prevLineOfCode) + prefix, + (nixCode.errPos.line - 1), + *nixCode.prevLineOfCode) << std::endl; } // line of code containing the error.%2$+5d% std::cout << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line), - nixCode.errLineOfCode) + prefix, + (nixCode.errPos.line), + nixCode.errLineOfCode) << std::endl; // error arrows for the column range. @@ -69,17 +69,17 @@ void printCodeLines(const string &prefix, const NixCode &nixCode) std::string arrows("^"); std::cout << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, - prefix, - spaces, - arrows) << std::endl; + prefix, + spaces, + arrows) << std::endl; } // next line of code. if (nixCode.nextLineOfCode.has_value()) { std::cout << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line + 1), - *nixCode.nextLineOfCode) + prefix, + (nixCode.errPos.line + 1), + *nixCode.nextLineOfCode) << std::endl; } } @@ -149,18 +149,18 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) // divider. if (einfo.name != "") out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, - prefix, - levelString, - einfo.name, - dashes, - einfo.programName.value_or("")) + prefix, + levelString, + einfo.name, + dashes, + einfo.programName.value_or("")) << std::endl; else out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, - prefix, - levelString, - dashes, - einfo.programName.value_or("")) + prefix, + levelString, + dashes, + einfo.programName.value_or("")) << std::endl; // filename. @@ -171,9 +171,9 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) : ""; out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL, - prefix, - einfo.nixCode->errPos.file, - eline) << std::endl; + prefix, + einfo.nixCode->errPos.file, + eline) << std::endl; out << prefix << std::endl; } else { out << fmt("%1%from command line argument", prefix) << std::endl; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 4a8b98640..e1750eb2a 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -60,7 +60,7 @@ public: void logEI(const ErrorInfo & ei) override { - std::stringstream oss; + std::stringstream oss; oss << ei; log(ei.level, oss.str()); @@ -68,7 +68,7 @@ public: void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) - override + override { if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); @@ -111,8 +111,7 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } -struct JSONLogger : Logger -{ +struct JSONLogger : Logger { Logger & prevLogger; JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } @@ -155,28 +154,26 @@ struct JSONLogger : Logger json["level"] = ei.level; json["msg"] = oss.str(); - // Extra things that COULD go into json. Useful? + // Extra things that COULD go into json. Useful? // TODO: decide if useful. // TODO: make a json obj that goes into json["msg"]? json["name"] = ei.name; json["description"] = ei.description; - if (ei.hint.has_value()) - { - json["hint"] = ei.hint->str(); + if (ei.hint.has_value()) { + json["hint"] = ei.hint->str(); } - if (ei.nixCode.has_value()) - { - if (ei.nixCode->errPos.line != 0) - json["line"] = ei.nixCode->errPos.line; - if (ei.nixCode->errPos.column != 0) - json["column"] = ei.nixCode->errPos.column; - if (ei.nixCode->errPos.file != "") - json["file"] = ei.nixCode->errPos.file; - if (ei.nixCode->prevLineOfCode.has_value()) - json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode; - json["errLineOfCode"] = ei.nixCode->errLineOfCode; - if (ei.nixCode->nextLineOfCode.has_value()) - json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode; + if (ei.nixCode.has_value()) { + if (ei.nixCode->errPos.line != 0) + json["line"] = ei.nixCode->errPos.line; + if (ei.nixCode->errPos.column != 0) + json["column"] = ei.nixCode->errPos.column; + if (ei.nixCode->errPos.file != "") + json["file"] = ei.nixCode->errPos.file; + if (ei.nixCode->prevLineOfCode.has_value()) + json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode; + json["errLineOfCode"] = ei.nixCode->errLineOfCode; + if (ei.nixCode->nextLineOfCode.has_value()) + json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode; } write(json); @@ -278,7 +275,8 @@ bool handleJSONLogMessage(const std::string & msg, return true; } -Activity::~Activity() { +Activity::~Activity() +{ try { logger.stopActivity(id); } catch (...) { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 89fd98419..0c4980b83 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -62,12 +62,12 @@ public: virtual void logEI(const ErrorInfo &ei) = 0; - void logEI(Verbosity lvl, ErrorInfo ei) + void logEI(Verbosity lvl, ErrorInfo ei) { ei.level = lvl; logEI(ei); } - + virtual void warn(const std::string & msg); virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, From 7afcb5af988eb0ce73c9916b809f8528dfb14c0f Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 18:42:19 +0200 Subject: [PATCH 166/350] Remove the `drain` argument from `readFile` Now it is always `drain` (see previous commit). --- src/libstore/gc.cc | 4 ++-- src/libutil/serialise.hh | 3 +++ src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 2 +- src/nix/ls.cc | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6bab1e37c..3cd35c945 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -419,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); + auto mapLines = tokenizeString>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) @@ -427,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile, true); + auto envString = readFile(envFile); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5780c93a6..a04118512 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -148,6 +148,9 @@ struct StringSink : Sink { ref s; StringSink() : s(make_ref()) { }; + explicit StringSink(const size_t reservedSize) : s(make_ref()) { + s->reserve(reservedSize); + }; StringSink(ref s) : s(s) { }; void operator () (const unsigned char * data, size_t len) override; }; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d001bc3b7..bb4af747b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -316,12 +316,12 @@ string readFile(int fd) } -string readFile(const Path & path, bool drain) +string readFile(const Path & path) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) throw SysError(format("opening file '%1%'") % path); - return drain ? drainFD(fd.get()) : readFile(fd.get()); + return readFile(fd.get()); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d7ee62bcc..32ef9a79a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -103,7 +103,7 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path, bool drain = false); +string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8590199d7..ee308a6ec 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -136,7 +136,7 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(make_ref(readFile(narPath, true)))); + list(makeNarAccessor(make_ref(readFile(narPath)))); } }; From 2e5be2a7495ac0b204454c74664e590a38d039d3 Mon Sep 17 00:00:00 2001 From: Guillaume Bouchard Date: Wed, 29 Apr 2020 18:44:01 +0200 Subject: [PATCH 167/350] StringSink pre allocate When used with `readFile`, we have a pretty good heuristic of the file size, so `reserve` this in the `string`. This will save some allocation / copy when the string is growing. --- src/libutil/util.cc | 10 +++++++--- src/libutil/util.hh | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bb4af747b..71db92d77 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -312,7 +312,11 @@ unsigned char getFileType(const Path & path) string readFile(int fd) { - return drainFD(fd, true); + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); } @@ -658,9 +662,9 @@ void writeFull(int fd, const string & s, bool allowInterrupts) } -string drainFD(int fd, bool block) +string drainFD(int fd, bool block, const size_t reserveSize) { - StringSink sink; + StringSink sink(reserveSize); drainFD(fd, sink, block); return std::move(*sink.s); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 32ef9a79a..1b263abcc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -162,7 +162,7 @@ MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd, bool block = true); +string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); From 2d0f766a778ac790be1ab1918db7c319ac7e5792 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 29 Apr 2020 11:52:35 -0600 Subject: [PATCH 168/350] more style tweaks --- .uncrustify.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.uncrustify.cfg b/.uncrustify.cfg index 23305adcf..b28935dcc 100644 --- a/.uncrustify.cfg +++ b/.uncrustify.cfg @@ -97,7 +97,7 @@ sp_cpp_lambda_square_paren = ignore # ignore/add/remove/force # Add or remove space after the capture specification of a C++11 lambda with # no argument list is present, as in '[] { ... }'. -sp_cpp_lambda_square_brace = ignore # ignore/add/remove/force +sp_cpp_lambda_square_brace = remove # ignore/add/remove/force # Add or remove space after the argument list of a C++11 lambda, as in # '[](int x) { ... }'. @@ -279,11 +279,11 @@ sp_angle_word = ignore # ignore/add/remove/force # Add or remove space between '>' and '>' in '>>' (template stuff). # # Default: add -sp_angle_shift = add # ignore/add/remove/force +sp_angle_shift = ignore # ignore/add/remove/force # (C++11) Permit removal of the space between '>>' in 'foo >'. Note # that sp_angle_shift cannot remove the space without this option. -sp_permit_cpp11_shift = false # true/false +sp_permit_cpp11_shift = true # true/false # Add or remove space before '(' of control statements ('if', 'for', 'switch', # 'while', etc.). @@ -1157,7 +1157,7 @@ indent_label = 1 # number # <=0: Subtract from brace indent # # Default: 1 -indent_access_spec = 1 # number +indent_access_spec = -4 # number # Whether to indent the code after an access specifier by one level. # If true, this option forces 'indent_access_spec=0'. @@ -1333,7 +1333,7 @@ nl_cs_property_leave_one_liners = true # true/false nl_func_leave_one_liners = true # true/false # Don't split one-line C++11 lambdas, as in '[]() { return 0; }'. -nl_cpp_lambda_leave_one_liners = false # true/false +nl_cpp_lambda_leave_one_liners = true # true/false # Don't split one-line if/else statements, as in 'if(...) b++;'. nl_if_leave_one_liners = true # true/false From d1229859c2612587bfd04111bae6584c0c9df051 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:48:29 +0200 Subject: [PATCH 169/350] Fix displaying error-position in `builtins.fetch{Tree,Tarball}` Without dereferencing this pointer, you'd get an error like this: ``` error: unsupported argument 'abc' to 'fetchTarball', at 0x13627e8 ``` --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 43c58485a..c5a0d9886 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -108,7 +108,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", - attr.name, who, attr.pos); + attr.name, who, *attr.pos); } if (!url) From 39ff80d031c5c9c1831ba5ea597c0de0181bfe34 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 29 Apr 2020 18:57:05 -0600 Subject: [PATCH 170/350] errorinfo constructor test --- src/error-demo/error-demo.cc | 10 ++++++++++ src/libstore/binary-cache-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libutil/error.hh | 7 ++++--- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 437a761c4..2c4f2da6e 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -23,6 +23,16 @@ int main() logger->logEI(e.info()); } + + // ErrorInfo constructor + try { + auto e = Error("generic error"); + throw DemoError(e.info()); + } catch (Error &e) { + logger->logEI(e.info()); + } + + // For completeness sake, info through vomit levels. // But this is maybe a heavy format for those. logger->logEI( diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4b7385c6b..97e34a75d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -284,7 +284,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.what()); + throw SubstituteGone(e.info()); } decompressor->finish(); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0f8126aee..cc336e460 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -365,7 +365,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); + throw InvalidPath(e.info()); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 03e43241f..48e6311bd 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -83,6 +83,7 @@ class BaseError : public std::exception protected: string prefix_; // used for location traces etc. ErrorInfo err; + std::optional what_; const string& calcWhat() { @@ -107,18 +108,18 @@ public: .hint = hintfmt(args...) } , status(status) - { } + { } template BaseError(const Args & ... args) : err { .level = lvlError, .hint = hintfmt(args...) } - { } + { } BaseError(ErrorInfo e) : err(e) - { } + { } virtual const char* sname() const { return "BaseError"; } From 2fcfc6c2c6ff144131d669ebfb2caa63c8b41417 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 13:05:29 +0200 Subject: [PATCH 171/350] nix dev-shell: Refactor script for getting the environment --- src/nix/get-env.sh | 8 ++++++++ src/nix/local.mk | 2 ++ src/nix/shell.cc | 18 ++++++++---------- 3 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 src/nix/get-env.sh diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh new file mode 100644 index 000000000..782f97b75 --- /dev/null +++ b/src/nix/get-env.sh @@ -0,0 +1,8 @@ +set -e +export IN_NIX_SHELL=impure +export dontAddDisableDepTrack=1 +if [[ -n $stdenv ]]; then + source $stdenv/setup +fi +export > $out +set >> $out diff --git a/src/nix/local.mk b/src/nix/local.mk index 033675e89..15cef55bb 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -27,3 +27,5 @@ $(foreach name, \ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +src/nix/shell.cc: src/nix/get-env.sh.gen.hh diff --git a/src/nix/shell.cc b/src/nix/shell.cc index fbfe24bbe..c8bf35a13 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -83,6 +83,10 @@ BuildEnvironment readEnvironment(const Path & path) return res; } +const static std::string getEnvSh = + #include "get-env.sh.gen.hh" + ; + /* Given an existing derivation, return the shell environment as initialised by stdenv's setup script. We do this by building a modified derivation with the same dependencies and nearly the same @@ -94,16 +98,9 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); - drv.args = { - "-c", - "set -e; " - "export IN_NIX_SHELL=impure; " - "export dontAddDisableDepTrack=1; " - "if [[ -n $stdenv ]]; then " - " source $stdenv/setup; " - "fi; " - "export > $out; " - "set >> $out "}; + auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + + drv.args = {store->printStorePath(getEnvShPath)}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); @@ -120,6 +117,7 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase(output.first); drv.env["out"] = ""; drv.env["outputs"] = "out"; + drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); From efe6c186eabeb85830be7bef9aea4dd7eb2357e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:39:26 +0200 Subject: [PATCH 172/350] nix dev-shell: Support structured attrs Tested against https://github.com/NixOS/nixpkgs/pull/72074. Fixes #3540. --- src/nix/get-env.sh | 1 + src/nix/shell.cc | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 782f97b75..a25ec43a9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,4 +1,5 @@ set -e +if [ -e .attrs.sh ]; then source .attrs.sh; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 if [[ -n $stdenv ]]; then diff --git a/src/nix/shell.cc b/src/nix/shell.cc index c8bf35a13..f0ffceaa7 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -13,7 +13,8 @@ using namespace nix; struct Var { - bool exported; + bool exported = true; + bool associative = false; std::string value; // quoted string or array }; @@ -48,11 +49,17 @@ BuildEnvironment readEnvironment(const Path & path) static std::string quotedStringRegex = R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; - static std::string arrayRegex = - R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + static std::string indexedArrayRegex = + R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + + /* Note: we distinguish between an indexed and associative array + using the space before the closing parenthesis. Will + undoubtedly regret this some day. */ + static std::regex assocArrayRegex( + "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); static std::regex functionRegex( "^" + varNameRegex + " \\(\\) *\n"); @@ -68,7 +75,12 @@ BuildEnvironment readEnvironment(const Path & path) else if (std::regex_search(pos, file.cend(), match, varRegex)) { pos = match[0].second; - res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); } else if (std::regex_search(pos, file.cend(), match, functionRegex)) { @@ -92,8 +104,10 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref store, Derivation drv) +StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) { + auto drv = store->derivationFromPath(drvPath); + auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); @@ -108,11 +122,12 @@ StorePath getDerivationEnvironment(ref store, Derivation drv) drv.env.erase("disallowedReferences"); drv.env.erase("disallowedRequisites"); - // FIXME: handle structured attrs - /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = drv.env["name"] + "-env"; + auto drvName = std::string(drvPath.name()); + assert(hasSuffix(drvName, ".drv")); + drvName.resize(drvName.size() - 4); + drvName += "-env"; for (auto & output : drv.outputs) drv.env.erase(output.first); drv.env["out"] = ""; @@ -161,9 +176,13 @@ struct Common : InstallableCommand, MixProfile for (auto & i : buildEnvironment.env) { if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - out << fmt("%s=%s\n", i.first, i.second.value); - if (i.second.exported) - out << fmt("export %s\n", i.first); + if (i.second.associative) + out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + else { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } } } @@ -194,7 +213,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + return getDerivationEnvironment(store, drvPath); } } From 0135fd6ec4dcd15771008b867f28047e5c2bb605 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Apr 2020 14:46:51 +0200 Subject: [PATCH 173/350] nix dev-shell: Unset shellHook This avoids inheriting the caller's shellHook, which can happen when running a dev-shell inside a dev-shell. --- src/nix/shell.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index f0ffceaa7..9c45a935d 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -172,6 +172,8 @@ struct Common : InstallableCommand, MixProfile void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { + out << "unset shellHook\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; for (auto & i : buildEnvironment.env) { From a15f918cbac51dee2e82b4cb0535e7ae5f4ff587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 14:57:40 +0200 Subject: [PATCH 174/350] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 +++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..5c9e1d9cc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Steps To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**`nix-env --version` output: ** + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..392ed30c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: improvement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 625868b33dab37c0750081843be8091e22808376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 30 Apr 2020 15:01:23 +0200 Subject: [PATCH 175/350] Update issue templates --- .github/ISSUE_TEMPLATE.md | 27 --------------------------- .github/ISSUE_TEMPLATE/bug_report.md | 10 ++++++++-- 2 files changed, 8 insertions(+), 29 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3372b1f03..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5c9e1d9cc..e6d346bc1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,19 +8,25 @@ assignees: '' --- **Describe the bug** + A clear and concise description of what the bug is. +If you have a problem with a specific package or NixOS, +you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. + **Steps To Reproduce** -Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error **Expected behavior** + A clear and concise description of what you expected to happen. -**`nix-env --version` output: ** +**`nix-env --version` output** **Additional context** + Add any other context about the problem here. From 171b4ce85c491563004ad6b74377a8cb70a7d06c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 30 Apr 2020 09:57:01 -0600 Subject: [PATCH 176/350] typo --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 71e640667..e0546ef78 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -219,7 +219,7 @@ struct CmdDevShell : Common, MixEnvironment mkFlag() .longName("command") .shortName('c') - .description("command and arguments to be executed insted of an interactive shell") + .description("command and arguments to be executed instead of an interactive shell") .labels({"command", "args"}) .arity(ArityAny) .handler([&](std::vector ss) { From 533343628dcd4cacc8ff74d429372bfbd860a683 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 30 Apr 2020 22:01:28 +0200 Subject: [PATCH 177/350] Revamp README.md --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9c0c87887..9aaca6ed3 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,55 @@ +# Nix + [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) -Nix, the purely functional package manager ------------------------------------------- +Nix is a powerful package manager for Linux and other Unix systems that makes package +management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +for more details. -Nix is a new take on package management that is fairly unique. Because of its -purity aspects, a lot of issues found in traditional package managers don't -appear with Nix. +## Installation -To find out more about the tool, usage and installation instructions, please -read the manual, which is available on the Nix website at -. +On Linux and macOS the easiest way to Install Nix is to run the following shell command +(as a user other than root): -## Contributing +``` +$ curl -L https://nixos.org/nix/install | sh +``` -Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking) -of the manual. It helps you to get started with building Nix from source. +Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). + +## Building And Developing + +### Building Nix + +You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): + +``` +$ nix-build ./release.nix -A build.aarch64-linux +$ nix-build ./release.nix -A build.x86_64-darwin +$ nix-build ./release.nix -A build.i686-linux +$ nix-build ./release.nix -A build.x86_64-linux +``` + +### Development Environment + +You can use the provided `shell.nix` to easily bootstrap working development environment: + +``` +$ nix-shell +$ ./bootstrap.sh +$ ./configure +$ make +``` + +## Additional Resources + +- [Nix manual](https://nixos.org/nix/manual) +- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix - A One Pager](https://github.com/tazjin/nix-1p) +- [NixOS Discourse](https://discourse.nixos.org/) +- #nix / #nixos on irc.freenode.net ## License -Nix is released under the LGPL v2.1 +Nix is released under the [LGPL v2.1](./COPYING) From f5d3215c8740feec8053a697eca04c12eb71191b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 30 Apr 2020 16:31:47 -0600 Subject: [PATCH 178/350] logError --- src/error-demo/error-demo.cc | 2 -- src/nix/verify.cc | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 2c4f2da6e..514258416 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -65,7 +65,6 @@ int main() .description = "Vomit description", }); - // Error in a program; no hint and no nix code. logError( ErrorInfo { .name = "name", @@ -80,7 +79,6 @@ int main() .hint = hintfmt("there was a %1%", "warning"), }); - // Warning with nix file, line number, column, and the lines of // code where a warning occurred. SymbolTable testTable; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b0658803..6e043dc2d 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -137,7 +137,7 @@ struct CmdVerify : StorePathsCommand doSigs(info2->sigs); } catch (InvalidPath &) { } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logError(e.info()); } } @@ -156,7 +156,7 @@ struct CmdVerify : StorePathsCommand done++; } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logError(e.info()); failed++; } From a3030e3c3186f980716c475155c387bf18041a3f Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 30 Apr 2020 17:56:26 -0600 Subject: [PATCH 179/350] fix error calls --- src/nix-channel/nix-channel.cc | 30 ++++++++++-------------------- src/nix-store/nix-store.cc | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index c403e3780..589a061c7 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -37,8 +37,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw Error(""); - // throw SysError("opening '%1%' for writing", channelsList); + throw SysError("opening '%1%' for writing", channelsList); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -47,11 +46,9 @@ static void writeChannels() static void addChannel(const string & url, const string & name) { if (!regex_search(url, std::regex("^(file|http|https)://"))) - throw Error(""); - // throw Error("invalid channel URL '%1%'", url); + throw Error("invalid channel URL '%1%'", url); if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) - throw Error(""); - // throw Error("invalid channel identifier '%1%'", name); + throw Error("invalid channel identifier '%1%'", name); readChannels(); channels[name] = url; writeChannels(); @@ -143,11 +140,9 @@ static void update(const StringSet & channelNames) if (S_ISLNK(st.st_mode)) // old-skool ~/.nix-defexpr if (unlink(nixDefExpr.c_str()) == -1) - throw Error(""); - // throw SysError("unlinking %1%", nixDefExpr); + throw SysError("unlinking %1%", nixDefExpr); } else if (errno != ENOENT) { - throw Error(""); - // throw SysError("getting status of %1%", nixDefExpr); + throw SysError("getting status of %1%", nixDefExpr); } createDirs(nixDefExpr); auto channelLink = nixDefExpr + "/channels"; @@ -199,12 +194,10 @@ static int _main(int argc, char ** argv) switch (cmd) { case cNone: - throw Error(""); - // throw UsageError("no command specified"); + throw UsageError("no command specified"); case cAdd: if (args.size() < 1 || args.size() > 2) - throw Error(""); - // throw UsageError("'--add' requires one or two arguments"); + throw UsageError("'--add' requires one or two arguments"); { auto url = args[0]; std::string name; @@ -220,14 +213,12 @@ static int _main(int argc, char ** argv) break; case cRemove: if (args.size() != 1) - throw Error(""); - // throw UsageError("'--remove' requires one argument"); + throw UsageError("'--remove' requires one argument"); removeChannel(args[0]); break; case cList: if (!args.empty()) - throw Error(""); - // throw UsageError("'--list' expects no arguments"); + throw UsageError("'--list' expects no arguments"); readChannels(); for (const auto & channel : channels) std::cout << channel.first << ' ' << channel.second << '\n'; @@ -237,8 +228,7 @@ static int _main(int argc, char ** argv) break; case cRollback: if (args.size() > 1) - throw Error(""); - // throw UsageError("'--rollback' has at most one argument"); + throw UsageError("'--rollback' has at most one argument"); Strings envArgs{"--profile", profile}; if (args.size() == 1) { envArgs.push_back("--switch-generation"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e4dd9bc96..1e8a88a7b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -906,7 +906,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - auto drvPath = store->parseStorePath(readString(in)); // inonal ony + auto drvPath = store->parseStorePath(readString(in)); // informational only BasicDerivation drv; readDerivation(in, *store, drv); From 536512d273f1d5821f65e9df77fe8f47b823ce6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:17:38 +0200 Subject: [PATCH 180/350] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9aaca6ed3..454b0ffa0 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap working development environment: +You can use the provided `shell.nix` to easily bootstrap a working development environment: ``` $ nix-shell From 30616d8e86f7ef635de561905eab0022000547b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:19:18 +0200 Subject: [PATCH 181/350] Update README.md Co-authored-by: worldofpeace --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 454b0ffa0..0f2f9bd1a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ $ make ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) -- [Nix jobs on nixos.hydra.org](https://hydra.nixos.org/project/nix) +- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - #nix / #nixos on irc.freenode.net From 2a434fc62bdb4391620d8030aec6b8d4a9f8ba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 1 May 2020 11:20:39 +0200 Subject: [PATCH 182/350] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f2f9bd1a..44687a150 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ $ make - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) -- #nix / #nixos on irc.freenode.net +- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License From 404a94ab6914405948640bedb92d1f9ddd502a8a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 May 2020 11:47:26 +0200 Subject: [PATCH 183/350] Tweak README.md --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 44687a150..a1588284d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Information on additional installation methods is available on the [Nix download ### Building Nix -You can build Nix via Nix via one of the targets provided by [release.nix](./release.nix): +You can build Nix using one of the targets provided by [release.nix](./release.nix): ``` $ nix-build ./release.nix -A build.aarch64-linux @@ -33,7 +33,7 @@ $ nix-build ./release.nix -A build.x86_64-linux ### Development Environment -You can use the provided `shell.nix` to easily bootstrap a working development environment: +You can use the provided `shell.nix` to get a working development environment: ``` $ nix-shell @@ -46,10 +46,9 @@ $ make - [Nix manual](https://nixos.org/nix/manual) - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [Nix - A One Pager](https://github.com/tazjin/nix-1p) - [NixOS Discourse](https://discourse.nixos.org/) - [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License -Nix is released under the [LGPL v2.1](./COPYING) +Nix is released under the [LGPL v2.1](./COPYING). From 5eebc4ad1d3965096b57c574d361d9274a7daafb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Mar 2020 13:39:39 +0100 Subject: [PATCH 184/350] Fix coverage job (cherry picked from commit bc5d4843a916dc4f99516e187a63108e8006f81f) --- release.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release.nix b/release.nix index 65b035957..f5729cee3 100644 --- a/release.nix +++ b/release.nix @@ -241,6 +241,11 @@ let src = nix; + preConfigure = + '' + ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor + ''; + enableParallelBuilding = true; buildInputs = buildDeps ++ propagatedDeps; From d6b4047c2fd8c197bf7a9f0f9a2f1cfb214e2e89 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 29 Apr 2020 14:02:37 -0600 Subject: [PATCH 185/350] rename nix run to nix shell and nix app to nix run (cherry picked from commit 5d8504b9789ffebabe8226227c4061dd48354177) --- src/nix/{shell.cc => dev-shell.cc} | 0 src/nix/run.cc | 14 +++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) rename src/nix/{shell.cc => dev-shell.cc} (100%) diff --git a/src/nix/shell.cc b/src/nix/dev-shell.cc similarity index 100% rename from src/nix/shell.cc rename to src/nix/dev-shell.cc diff --git a/src/nix/run.cc b/src/nix/run.cc index 5334531fd..ebfec36d9 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -57,11 +57,11 @@ struct RunCommon : virtual Command } }; -struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment +struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment { std::vector command = { getEnv("SHELL").value_or("bash") }; - CmdRun() + CmdShell() { mkFlag() .longName("command") @@ -85,19 +85,19 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment return { Example{ "To start a shell providing GNU Hello from NixOS 17.03:", - "nix run -f channel:nixos-17.03 hello" + "nix shell -f channel:nixos-17.03 hello" }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix run nixpkgs.youtube-dl" + "nix shell nixpkgs.youtube-dl" }, Example{ "To run GNU Hello:", - "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix run --store ~/my-nix nixpkgs.hello -c hello" + "nix shell --store ~/my-nix nixpkgs.hello -c hello" }, }; } @@ -141,7 +141,7 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment } }; -static auto r1 = registerCommand("run"); +static auto r1 = registerCommand("shell"); void chrootHelper(int argc, char * * argv) { From 2852a486f864c385696fb7ee28836b183976ca63 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Wed, 29 Apr 2020 15:37:21 -0600 Subject: [PATCH 186/350] rename run to shell in tests (cherry picked from commit f459ca547fda4b0ca477d895413e86b69a580ab0) --- tests/local.mk | 2 +- tests/run.sh | 28 ---------------------------- tests/{run.nix => shell-hello.nix} | 0 tests/shell.sh | 28 ++++++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 29 deletions(-) delete mode 100644 tests/run.sh rename tests/{run.nix => shell-hello.nix} (100%) create mode 100644 tests/shell.sh diff --git a/tests/local.mk b/tests/local.mk index c5602773d..56e5640ca 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -21,7 +21,7 @@ nix_tests = \ fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ - run.sh \ + shell.sh \ brotli.sh \ pure-eval.sh \ check.sh \ diff --git a/tests/run.sh b/tests/run.sh deleted file mode 100644 index d1dbfd6bd..000000000 --- a/tests/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -source common.sh - -clearStore -clearCache - -nix run -f run.nix hello -c hello | grep 'Hello World' -nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS' - -if ! canUseSandbox; then exit; fi - -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 - -clearStore - -path=$(nix eval --raw -f run.nix hello) - -# Note: we need the sandbox paths to ensure that the shell is -# visible in the sandbox. -nix run --sandbox-build-dir /build-tmp \ - --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ - --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World' - -path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello') - -[[ $path/bin/hello = $path2 ]] - -[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/run.nix b/tests/shell-hello.nix similarity index 100% rename from tests/run.nix rename to tests/shell-hello.nix diff --git a/tests/shell.sh b/tests/shell.sh new file mode 100644 index 000000000..7a9ee8ab0 --- /dev/null +++ b/tests/shell.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore +clearCache + +nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' + +if ! canUseSandbox; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f shell-hello.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix shell --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] From c089c52d5f1cff888552f485775b74226dcbe618 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 May 2020 12:42:39 +0200 Subject: [PATCH 187/350] Fix build --- src/nix/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/local.mk b/src/nix/local.mk index 15cef55bb..8c0eed19e 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/shell.cc: src/nix/get-env.sh.gen.hh +src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh From 4b99c09f5ccd385d2bf0c82a8c9a4ae1658abbe8 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 1 May 2020 14:32:06 -0600 Subject: [PATCH 188/350] convert some errors --- src/libstore/builtins/buildenv.cc | 12 +++++++-- src/libstore/download.cc | 7 ++++- src/libstore/gc.cc | 13 ++++++--- src/libstore/local-store.cc | 44 +++++++++++++++++++++++-------- src/libstore/optimise-store.cc | 18 ++++++++++--- src/libstore/remote-store.cc | 9 +++++++ src/libstore/sqlite.cc | 5 +++- 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 000769094..64085fcc9 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,7 +22,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + logWarning( + ErrorInfo { + .name = "Create Links - Directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + }); return; } throw; @@ -41,7 +45,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - printError("warning: skipping dangling symlink '%s'", dstFile); + logWarning( + ErrorInfo { + .name = "Create Links - Skipping Symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + }); continue; } throw; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 7d0cb449b..60c6a80f3 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -600,7 +600,12 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError("unexpected error in download thread: %s", e.what()); + logError( + ErrorInfo { + .name = "download", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); } { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 27bd3f3e9..4f6f78a99 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -41,7 +41,8 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); + // TODO: info? + printError("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -129,10 +130,14 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, if (settings.checkRootReachability) { auto roots = findRoots(false); if (roots[storePath.clone()].count(gcRoot) == 0) - printError( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " + + logWarning( + ErrorInfo { + .name = "GC Root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)); + gcRoot, printStorePath(storePath)) + }); } /* Grab the global GC root, causing us to block while a GC is in diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index db5a3b53c..1f16a22b1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,8 +87,13 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + logError( + ErrorInfo { + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) + }); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) @@ -876,7 +881,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) - printError(e.what()); + logError(e.info()); else throw; } @@ -1237,9 +1242,13 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - printError( + logError( + ErrorInfo { + .name = "Invalid Hash", + .hint = hintfmt( "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash); + linkPath, link.name, hash) + }); if (repair) { if (unlink(linkPath.c_str()) == 0) printError("removed link '%s'", linkPath); @@ -1272,8 +1281,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + logError( + ErrorInfo { + .name = "Invalid Hash - Path Modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(), current.first.to_string()) + }); if (repair) repairPath(i); else errors = true; } else { @@ -1304,7 +1317,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError("error: %s", e.msg()); + logError(e.info()); else warn(e.msg()); errors = true; @@ -1324,7 +1337,11 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); + logError( + ErrorInfo { + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) + }); return; } @@ -1347,12 +1364,17 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - printError("path '%s' disappeared, but it still has valid referrers!", pathS); + // TODO log as warning if repair successful?? + logError( + ErrorInfo { + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + }); if (repair) try { repairPath(path); } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); errors = true; } else errors = true; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index f9cf2bb6b..9f6112183 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -130,7 +130,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError("skipping suspicious writable file '%1%'", path); + logWarning( + ErrorInfo { + .name = "Suspicious File", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) + }); return; } @@ -194,7 +198,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - printError("removing corrupted link '%1%'", linkPath); + logWarning( + ErrorInfo { + .name = "Corrupted Link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) + }); unlink(linkPath.c_str()); goto retry; } @@ -229,7 +237,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError("unable to unlink '%1%'", tempLink); + logError( + ErrorInfo { + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) + }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cc336e460..7c50c1065 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -780,6 +780,15 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_NEXT) + // TODO: is this really an ErrorInfo error? Seems like we're forwarding the + // stderr output of the remote to current stderr/log + // ErrorInfo gets lost in this scenario. + // An alternative might be a logger on the remote that forwards ErrorInfo and etc. + // logError( + // ErrorInfo { + // // .name = "Remote Store" TODO reasonable name. + // .hint = hintfmt(chomp(readString(from))) + // }); printError(chomp(readString(from))); else if (msg == STDERR_START_ACTIVITY) { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a72cd5d88..082c54005 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -204,7 +204,10 @@ void handleSQLiteBusy(SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; - printError("warning: %s", e.what()); + logWarning( + ErrorInfo { .name = "sqlite busy", + .hint = hintfmt(e.what()) + }); } /* Sleep for a while since retrying the transaction right away From 2aeb874e832ae4c326f1115e55fdcaf7c2f4e916 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Fri, 1 May 2020 23:32:01 +0200 Subject: [PATCH 189/350] Improve help-message for nix-repl * Remove obsolete `printHelp` function * Add an example to demonstrate how to list all available commands within the REPL --- src/nix/repl.cc | 44 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 27727bd25..ea8ff1553 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -82,40 +82,6 @@ struct NixRepl : gc }; -void printHelp() -{ - std::cout - << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" - << "\n" - << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" - << "\n" - << "Options:\n" - << " --help\n" - << " Prints out a summary of the command syntax and exits.\n" - << "\n" - << " --version\n" - << " Prints out the Nix version number on standard output and exits.\n" - << "\n" - << " -I path\n" - << " Add a path to the Nix expression search path. This option may be given\n" - << " multiple times. See the NIX_PATH environment variable for information on\n" - << " the semantics of the Nix search path. Paths added through -I take\n" - << " precedence over NIX_PATH.\n" - << "\n" - << " paths...\n" - << " A list of paths to files containing Nix expressions which nix-repl will\n" - << " load and add to its scope.\n" - << "\n" - << " A path surrounded in < and > will be looked up in the Nix expression search\n" - << " path, as in the Nix language itself.\n" - << "\n" - << " If an element of paths starts with http:// or https://, it is interpreted\n" - << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" - << " location. The tarball must include a single top-level directory containing\n" - << " at least a file named default.nix.\n"; -} - - string removeWhitespace(string s) { s = chomp(s); @@ -809,6 +775,16 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } + Examples examples() override + { + return { + Example{ + "Display all special commands within the REPL:", + "nix repl\n nix-repl> :?" + } + }; + } + void run(ref store) override { auto repl = std::make_unique(searchPath, openStore()); From f16e24f95e7d7fa3a5c3b2c8d43d171fa5b9cf85 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 2 May 2020 10:58:33 +0200 Subject: [PATCH 190/350] remote-store: don't log raw stderr by default For remote stores the log messages are already forwarded as structured STDERR_RESULT messages so the old format is duplicate information. But still included with -vvv since it could be useful for debugging problems. $ nix build -L /nix/store/nl71b2niws857ffiaggyrkjwgx9jjzc0-foo.drv --store ssh-ng://localhost Hello World! foo> Hello World! [1/0/1 built] building foo Fixes #3556 --- src/libstore/remote-store.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..07ef79382 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -779,8 +779,10 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return std::make_exception_ptr(Error(status, error)); } - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); + else if (msg == STDERR_NEXT) { + string s = chomp(readString(from)); + printMsg(lvlVomit, "stderr %s", s); + } else if (msg == STDERR_START_ACTIVITY) { auto act = readNum(from); From 4769eea5e2229c01c3b5d050891b071a45a024d5 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 2 May 2020 23:26:12 +0200 Subject: [PATCH 191/350] logging: handle build log lines in simple logger The raw stderr output isn't logged anymore so the build logs need to be printed by the default logger in order for the old commands like nix-build to still show build output. --- src/libutil/logging.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 3cc4ef8f1..777650de5 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -63,6 +63,16 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } + void result(ActivityId act, ResultType type, const std::vector & fields) override + { + if (type == resBuildLogLine || type == resPostBuildLogLine) { + assert(0 < fields.size()); + assert(fields[0].type == Logger::Field::tString); + auto lastLine = fields[0].s; + log(lvlInfo, lastLine); + } + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override From ab6f0b9641ad6e9cef72f73d23e31138a97a225b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 3 May 2020 08:01:25 -0600 Subject: [PATCH 192/350] convert some printError calls to logError --- src/build-remote/build-remote.cc | 8 +- src/libexpr/parser.y | 15 ++- src/libmain/shared.cc | 10 +- src/libstore/build.cc | 97 ++++++++++++++----- src/libstore/pathlocks.cc | 3 +- src/libstore/s3-binary-cache-store.cc | 4 +- src/libstore/store-api.cc | 2 +- src/libutil/affinity.cc | 4 +- src/libutil/error.hh | 6 ++ src/libutil/logging.hh | 1 + src/libutil/util.cc | 18 ++-- src/nix-build/nix-build.cc | 2 + src/nix-daemon/nix-daemon.cc | 3 +- src/nix-env/nix-env.cc | 23 ++++- src/nix-env/user-env.cc | 2 +- src/nix-store/nix-store.cc | 15 ++- src/nix/upgrade-nix.cc | 8 +- src/nix/verify.cc | 20 +++- .../resolve-system-dependencies.cc | 24 ++++- 19 files changed, 195 insertions(+), 70 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 00340b787..0d6859596 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -200,9 +200,13 @@ static int _main(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - printError("cannot build on '%s': %s%s", + logError( + ErrorInfo { + .name = "remote build", + .hint = hintfmt("cannot build on '%s': %s%s", bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)); + (msg.empty() ? "" : ": " + msg)) + }); bestMachine->enabled = false; continue; } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3767532d5..3ed9a7a4f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -689,8 +689,12 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl request.unpack = true; res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { - // TODO: change to warn()? - printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second); + logWarning( + ErrorInfo { + .name = "Download Error", + .hint = hintfmt("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + }); + res = { false, "" }; } } else { @@ -698,8 +702,11 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (pathExists(path)) res = { true, path }; else { - // TODO: change to warn()? - printError("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second); + logWarning( + ErrorInfo { + .name = "Search path not found", + .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + }); res = { false, "" }; } } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8551eb048..5773d90cc 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -300,21 +300,21 @@ int handleExceptions(const string & programName, std::function fun) } catch (UsageError & e) { // TODO: switch to logError // logError(e.info()); - printError( + _printError( format("%1%\nTry '%2% --help' for more information.") % e.what() % programName); return 1; } catch (BaseError & e) { // logError(e.info()); - printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + _printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); if (e.prefix() != "" && !settings.showTrace) - printError("(use '--show-trace' to show detailed location information)"); + _printError("(use '--show-trace' to show detailed location information)"); return e.status; } catch (std::bad_alloc & e) { - printError(error + "out of memory"); + _printError(error + "out of memory"); return 1; } catch (std::exception & e) { - printError(error + e.what()); + _printError(error + e.what()); return 1; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2301adb16..bacbd5808 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -493,7 +493,9 @@ void handleDiffHook( if (diffRes.second != "") printError(chomp(diffRes.second)); } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); + // logError(error.info()) + // TODO append message onto errorinfo... + _printError("diff hook execution failed: %s", error.what()); } } } @@ -1144,7 +1146,11 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); + logError( + ErrorInfo { + .name = "missing derivation during build", + .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) + }); done(BuildResult::MiscFailure); return; } @@ -1295,8 +1301,14 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError("found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + logError( + ErrorInfo { + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + }); + auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); @@ -1330,8 +1342,13 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - printError("cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed); + logError( + ErrorInfo { + .name = "Dependencies could not be built", + .hint = hintfmt( + "cannot build derivation '%s': %s dependencies couldn't be built", + worker.store.printStorePath(drvPath), nrFailed) + }); done(BuildResult::DependencyFailed); return; } @@ -1489,7 +1506,7 @@ void DerivationGoal::tryToBuild() startBuilder(); } catch (BuildError & e) { - printError(e.msg()); + logError(e.info()); outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; @@ -1709,7 +1726,7 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - printError(e.msg()); + logError(e.info()); outputLocks.unlock(); @@ -1788,8 +1805,13 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - printError("build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); + logError( + ErrorInfo { + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) + }); worker.hook = 0; return rpDecline; } else @@ -3783,10 +3805,10 @@ void DerivationGoal::registerOutputs() result.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output '%s' of '%s' differs from '%s' from previous round", + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : fmt("output '%s' of '%s' differs from previous round", + : hintfmt("output '%s' of '%s' differs from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( @@ -3796,9 +3818,15 @@ void DerivationGoal::registerOutputs() worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) - throw NotDeterministic(msg); + throw NotDeterministic(hint); + + logError( + ErrorInfo { + .name = "Output determinism error", + .hint = hint + }); + - printError(msg); curRound = nrRounds; // we know enough, bail out early } } @@ -4060,9 +4088,13 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); + logError( + ErrorInfo { + .name = "Max log size exceeded", + .hint = hintfmt( + "%1% killed after writing more than %2% bytes of log output", + getName(), settings.maxLogSize) + }); killChild(); done(BuildResult::LogLimitExceeded); return; @@ -4352,7 +4384,7 @@ void SubstitutionGoal::tryNext() throw; } catch (Error & e) { if (settings.tryFallback) { - printError(e.what()); + logError(e.info()); tryNext(); return; } @@ -4864,9 +4896,13 @@ void Worker::waitForInput() j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); + logError( + ErrorInfo { + .name = "Silent build timeout", + .hint = hintfmt( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime) + }); goal->timedOut(); } @@ -4875,9 +4911,13 @@ void Worker::waitForInput() j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); + logError( + ErrorInfo { + .name = "Build timeout", + .hint = hintfmt( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout) + }); goal->timedOut(); } } @@ -4939,7 +4979,12 @@ bool Worker::pathContentsGood(const StorePath & path) res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path.clone(), res); - if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); + if (!res) + logError( + ErrorInfo { + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + }); return res; } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 52d430ffd..926f4ea1e 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -160,7 +160,8 @@ void PathLocks::unlock() if (close(i.first) == -1) printError( - format("error (ignored): cannot close lock file on '%1%'") % i.second); + "error (ignored): cannot close lock file on '%1%'", + i.second); debug(format("lock released on '%1%'") % i.second); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 0326821f6..3175eb69b 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -111,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + error.GetExceptionName(), + error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a04d5013e..542e5c552 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -961,7 +961,7 @@ std::list> getDefaultSubstituters() try { stores.push_back(openStore(uri)); } catch (Error & e) { - printError("warning: %s", e.what()); + logWarning(e.info()); } }; diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index 98f8287ad..e3cf33b58 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -25,7 +25,7 @@ void setAffinityTo(int cpu) CPU_ZERO(&newAffinity); CPU_SET(cpu, &newAffinity); if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError(format("failed to lock thread to CPU %1%") % cpu); + printError("failed to lock thread to CPU %1%", cpu); #endif } @@ -47,7 +47,7 @@ void restoreAffinity() #if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - printError("failed to restore affinity %1%"); + _printError("failed to restore affinity"); #endif } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 48e6311bd..86cff5609 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -117,6 +117,12 @@ public: } { } + BaseError(hintformat hint) + : err { .level = lvlError, + .hint = hint + } + { } + BaseError(ErrorInfo e) : err(e) { } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 0c4980b83..5f03cdf41 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -149,6 +149,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ } \ } while (0) +#define _printError(args...) printMsg(lvlError, args) #define printError(args...) printMsg(lvlError, args) #define printInfo(args...) printMsg(lvlInfo, args) #define printTalkative(args...) printMsg(lvlTalkative, args) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 747a9e991..68dc1b738 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1417,15 +1417,15 @@ string base64Decode(const string & s) } -void callFailure(const std::function & failure, std::exception_ptr exc) -{ - try { - failure(exc); - } catch (std::exception & e) { - printError("uncaught exception: %s", e.what()); - abort(); - } -} +// void callFailure(const std::function & failure, std::exception_ptr exc) +// { +// try { +// failure(exc); +// } catch (std::exception & e) { +// printError("uncaught exception: %s", e.what()); +// abort(); +// } +// } static Sync> windowSize{{0, 0}}; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 0a058a31b..401c8d340 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -368,6 +368,8 @@ static void _main(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { + // TODO: append error msg + logError(e.info()); printError("warning: %s; will use bash from your environment", e.what()); shell = "bash"; } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5f78ab464..3cd2c3a6b 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -247,7 +247,8 @@ static void daemonLoop(char * * argv) } catch (Interrupted & e) { return; } catch (Error & e) { - printError("error processing connection: %1%", e.msg()); + // TODO append error message + _printError("error processing connection: %1%", e.msg()); } } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index dde8875f1..76008d00c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -123,7 +123,11 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); + logError( + ErrorInfo { + .name = "Name collision", + .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) + }); continue; } /* Load the expression on demand. */ @@ -860,7 +864,12 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + logError( + ErrorInfo { + .name = "Invalid meta attribute", + .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); placeholder.write(nullptr); } else { PathSet context; @@ -1110,8 +1119,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) XMLAttrs attrs2; attrs2["name"] = j; Value * v = i.queryMeta(j); - if (!v) - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + if (!v) + logError( + ErrorInfo { + .name = "Invalid meta attribute", + .hint = hintfmt( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); else { if (v->type == tString) { attrs2["type"] = "string"; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index f852916d8..2484b9759 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -146,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Path lockTokenCur = optimisticLockProfile(profile); if (lockToken != lockTokenCur) { - printError(format("profile '%1%' changed while we were busy; restarting") % profile); + printError("profile '%1%' changed while we were busy; restarting", profile); return false; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 1e8a88a7b..2e2276b4a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -726,9 +726,15 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - printError( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); + logError( + ErrorInfo { + .name = "Hash match error", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(), + current.first.to_string()) + }); status = 1; } } @@ -835,7 +841,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & p : willSubstitute) subs.emplace_back(p.clone()); store->buildPaths(subs); } catch (Error & e) { - printError("warning: %1%", e.msg()); + // logWarning(e.info()) TODO: + _printError("warning: %1%", e.msg()); } } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index c05c29517..8f4e529bc 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -64,7 +64,13 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - printError("would upgrade to version %s", version); + // TODO change to info? + logWarning( + ErrorInfo { + .name = "Version update", + .hint = hintfmt("would upgrade to version %s", version) + }); + // printError("would upgrade to version %s", version); return; } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6e043dc2d..f53217239 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -97,9 +97,15 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - printError( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); + logError( + ErrorInfo { + .name = "Hash error - path modified", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(), + hash.first.to_string()) + }); } } @@ -148,7 +154,13 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - printError("path '%s' is untrusted", store->printStorePath(info->path)); + logError( + ErrorInfo { + .name = "Untrusted path", + .hint = hintfmt("path '%s' is untrusted", + store->printStorePath(info->path)) + }); + } } diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 8f0c99c84..dcea72529 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,12 +39,20 @@ std::set runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular file", filename); + logError( + ErrorInfo { + .name = "Regular MACH file", + .hint = hintfmt("file '%s' is not a regular file", filename) + }); return {}; } if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); + logError( + ErrorInfo { + .name = "File too short", + .hint = hintfmt("file '%s' is too short for a MACH binary", filename) + }); return {}; } @@ -66,13 +74,21 @@ std::set runResolver(const Path & filename) } } if (mach64_offset == 0) { - printError(format("Could not find any mach64 blobs in file '%1%', continuing...") % filename); + logError( + ErrorInfo { + .name = "No mach64 blobs", + .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) + }); return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - printError(format("Object file has unknown magic number '%1%', skipping it...") % magic); + logError( + ErrorInfo { + .name = "Magic number", + .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) + }); return {}; } From c05f0e3093d4fa4ea9b07b038099ff9555258717 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 12:28:28 -0600 Subject: [PATCH 193/350] closer but still lambda indent problems --- .uncrustify.cfg | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.uncrustify.cfg b/.uncrustify.cfg index b28935dcc..6bf91ec3c 100644 --- a/.uncrustify.cfg +++ b/.uncrustify.cfg @@ -1041,14 +1041,14 @@ indent_else_if = false # true/false indent_var_def_blk = 0 # number # Whether to indent continued variable declarations instead of aligning. -indent_var_def_cont = false # true/false +indent_var_def_cont = true # true/false # Whether to indent continued shift expressions ('<<' and '>>') instead of # aligning. Set align_left_shift=false when enabling this. indent_shift = false # true/false # Whether to force indentation of function definitions to start in column 1. -indent_func_def_force_col1 = false # true/false +indent_func_def_force_col1 = true # true/false # Whether to indent continued function call parameters one indent level, # rather than aligning parameters under the open parenthesis. @@ -1118,7 +1118,7 @@ indent_sing_line_comments = 0 # unsigned number indent_relative_single_line_comments = false # true/false # Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns. -indent_switch_case = 0 # unsigned number +indent_switch_case = indent_columns # unsigned number # indent 'break' with 'case' from 'switch'. indent_switch_break_with_case = false # true/false @@ -1217,13 +1217,13 @@ indent_preserve_sql = false # true/false # followed by a newline, the next line is indent one tab. # # Default: true -indent_align_assign = true # true/false +indent_align_assign = false # true/false # Whether to align continued statements at the '('. If false or the '(' is # followed by a newline, the next line indent is one tab. # # Default: true -indent_align_paren = true # true/false +indent_align_paren = true # true/false, # (OC) Whether to indent Objective-C blocks at brace level instead of usual # rules. @@ -1299,7 +1299,8 @@ indent_ternary_operator = 0 # unsigned number # If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column. indent_off_after_return_new = false # true/false -# If true, the tokens after return are indented with regular single indentation. By default (false) the indentation is after the return token. +# If true, the tokens after return are indented with regular single indentation. +# By default (false) the indentation is after the return token. indent_single_after_return = false # true/false # Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they @@ -2902,7 +2903,7 @@ use_indent_continue_only_once = false # true/false # # true: indentation will be used only once # false: indentation will be used every time (default) -indent_cpp_lambda_only_once = false # true/false +indent_cpp_lambda_only_once = true # true/false # Whether sp_after_angle takes precedence over sp_inside_fparen. This was the # historic behavior, but is probably not the desired behavior, so this is off From 9c5ece44a7f32784dd5e2ea0faf110054f9233a8 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 13:46:15 -0600 Subject: [PATCH 194/350] separate msgs instead of appending to what() --- src/libmain/shared.cc | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5773d90cc..db1e0ba1d 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -298,23 +298,21 @@ int handleExceptions(const string & programName, std::function fun) } catch (Exit & e) { return e.status; } catch (UsageError & e) { - // TODO: switch to logError - // logError(e.info()); - _printError( - format("%1%\nTry '%2% --help' for more information.") - % e.what() % programName); + logError(e.info()); + printError("Try '%1% --help' for more information.", programName); return 1; } catch (BaseError & e) { - // logError(e.info()); - _printError("%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); + if (settings.showTrace && e.prefix() != "") + printError(e.prefix()); + logError(e.info()); if (e.prefix() != "" && !settings.showTrace) - _printError("(use '--show-trace' to show detailed location information)"); + printError("(use '--show-trace' to show detailed location information)"); return e.status; } catch (std::bad_alloc & e) { - _printError(error + "out of memory"); + printError(error + "out of memory"); return 1; } catch (std::exception & e) { - _printError(error + e.what()); + printError(error + e.what()); return 1; } From e9f10beed1a30fe80b315a9de4c4c2ab11527db8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2020 22:40:02 +0200 Subject: [PATCH 195/350] precompiled-headers.h: Don't include our own headers --- precompiled-headers.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/precompiled-headers.h b/precompiled-headers.h index e0d885b23..079aa496e 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -56,6 +56,3 @@ #include #include #include - -#include "util.hh" -#include "args.hh" From a721a0b1140bf489d645f5d85737acafc1c57c65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 May 2020 22:40:19 +0200 Subject: [PATCH 196/350] Flag: Use designated initializers --- src/libexpr/common-eval-args.cc | 35 ++++---- src/libmain/common-args.cc | 60 +++++++------- src/libmain/shared.cc | 53 ++++++------ src/libstore/globals.cc | 33 ++++---- src/libutil/args.cc | 44 +++++----- src/libutil/args.hh | 139 +++++++++++++++++--------------- src/libutil/config.cc | 35 ++++---- src/nix/add-to-store.cc | 13 +-- src/nix/build.cc | 22 ++--- src/nix/command.cc | 73 +++++++++-------- src/nix/copy.cc | 43 +++++----- src/nix/dev-shell.cc | 16 ++-- src/nix/hash.cc | 8 +- src/nix/installables.cc | 13 +-- src/nix/main.cc | 58 +++++++------ src/nix/run.cc | 16 ++-- src/nix/search.cc | 20 ++--- src/nix/show-derivation.cc | 11 +-- src/nix/sigs.cc | 27 ++++--- src/nix/upgrade-nix.cc | 24 +++--- src/nix/verify.cc | 14 ++-- src/nix/why-depends.cc | 11 +-- 22 files changed, 406 insertions(+), 362 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c9c16bb..44baadd53 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -10,24 +10,27 @@ namespace nix { MixEvalArgs::MixEvalArgs() { - mkFlag() - .longName("arg") - .description("argument to be passed to Nix functions") - .labels({"name", "expr"}) - .handler([&](std::vector ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); + addFlag({ + .longName = "arg", + .description = "argument to be passed to Nix functions", + .labels = {"name", "expr"}, + .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + }); - mkFlag() - .longName("argstr") - .description("string-valued argument to be passed to Nix functions") - .labels({"name", "string"}) - .handler([&](std::vector ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + addFlag({ + .longName = "argstr", + .description = "string-valued argument to be passed to Nix functions", + .labels = {"name", "string"}, + .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + }); - mkFlag() - .shortName('I') - .longName("include") - .description("add a path to the list of locations used to look up <...> file names") - .label("path") - .handler([&](std::string s) { searchPath.push_back(s); }); + addFlag({ + .longName = "include", + .shortName = 'I', + .description = "add a path to the list of locations used to look up <...> file names", + .labels = {"path"}, + .handler = {[&](std::string s) { searchPath.push_back(s); }} + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9e1d7cee6..51e199ea5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -6,43 +6,47 @@ namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) : programName(programName) { - mkFlag() - .longName("verbose") - .shortName('v') - .description("increase verbosity level") - .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); + addFlag({ + .longName = "verbose", + .shortName = 'v', + .description = "increase verbosity level", + .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, + }); - mkFlag() - .longName("quiet") - .description("decrease verbosity level") - .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); + addFlag({ + .longName = "quiet", + .description = "decrease verbosity level", + .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, + }); - mkFlag() - .longName("debug") - .description("enable debug output") - .handler([]() { verbosity = lvlDebug; }); + addFlag({ + .longName = "debug", + .description = "enable debug output", + .handler = {[]() { verbosity = lvlDebug; }}, + }); - mkFlag() - .longName("option") - .labels({"name", "value"}) - .description("set a Nix configuration option (overriding nix.conf)") - .arity(2) - .handler([](std::vector ss) { + addFlag({ + .longName = "option", + .description = "set a Nix configuration option (overriding nix.conf)", + .labels = {"name", "value"}, + .handler = {[](std::string name, std::string value) { try { - globalConfig.set(ss[0], ss[1]); + globalConfig.set(name, value); } catch (UsageError & e) { warn(e.what()); } - }); + }}, + }); - mkFlag() - .longName("max-jobs") - .shortName('j') - .label("jobs") - .description("maximum number of parallel builds") - .handler([=](std::string s) { + addFlag({ + .longName = "max-jobs", + .shortName = 'j', + .description = "maximum number of parallel builds", + .labels = Strings{"jobs"}, + .handler = {[=](std::string s) { settings.set("max-jobs", s); - }); + }} + }); std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8586257bf..70d1f0186 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -165,28 +165,32 @@ LegacyArgs::LegacyArgs(const std::string & programName, std::function parseArg) : MixCommonArgs(programName), parseArg(parseArg) { - mkFlag() - .longName("no-build-output") - .shortName('Q') - .description("do not show build output") - .set(&settings.verboseBuild, false); + addFlag({ + .longName = "no-build-output", + .shortName = 'Q', + .description = "do not show build output", + .handler = {&settings.verboseBuild, false}, + }); - mkFlag() - .longName("keep-failed") - .shortName('K') - .description("keep temporary directories of failed builds") - .set(&(bool&) settings.keepFailed, true); + addFlag({ + .longName = "keep-failed", + .shortName ='K', + .description = "keep temporary directories of failed builds", + .handler = {&(bool&) settings.keepFailed, true}, + }); - mkFlag() - .longName("keep-going") - .shortName('k') - .description("keep going after a build fails") - .set(&(bool&) settings.keepGoing, true); + addFlag({ + .longName = "keep-going", + .shortName ='k', + .description = "keep going after a build fails", + .handler = {&(bool&) settings.keepGoing, true}, + }); - mkFlag() - .longName("fallback") - .description("build from source if substitution fails") - .set(&(bool&) settings.tryFallback, true); + addFlag({ + .longName = "fallback", + .description = "build from source if substitution fails", + .handler = {&(bool&) settings.tryFallback, true}, + }); auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { @@ -205,11 +209,12 @@ LegacyArgs::LegacyArgs(const std::string & programName, mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", &gcWarning, false); - mkFlag() - .longName("store") - .label("store-uri") - .description("URI of the Nix store to use") - .dest(&(std::string&) settings.storeUri); + addFlag({ + .longName = "store", + .description = "URI of the Nix store to use", + .labels = {"store-uri"}, + .handler = {&(std::string&) settings.storeUri}, + }); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d6ea0318e..a0a2d850e 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -167,21 +167,24 @@ template<> void BaseSetting::toJSON(JSONPlaceholder & out) template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description("Enable sandboxing.") - .handler([=](std::vector ss) { override(smEnabled); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description("Disable sandboxing.") - .handler([=](std::vector ss) { override(smDisabled); }) - .category(category); - args.mkFlag() - .longName("relaxed-" + name) - .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](std::vector ss) { override(smRelaxed); }) - .category(category); + args.addFlag({ + .longName = name, + .description = "Enable sandboxing.", + .category = category, + .handler = {[=]() { override(smEnabled); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = "Disable sandboxing.", + .category = category, + .handler = {[=]() { override(smDisabled); }} + }); + args.addFlag({ + .longName = "relaxed-" + name, + .description = "Enable sandboxing, but allow builds to disable it.", + .category = category, + .handler = {[=]() { override(smRelaxed); }} + }); } void MaxBuildJobsSetting::set(const std::string & str) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ba15ea571..91fc2f581 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -3,16 +3,14 @@ namespace nix { -Args::FlagMaker Args::mkFlag() -{ - return FlagMaker(*this); -} - -Args::FlagMaker::~FlagMaker() +void Args::addFlag(Flag && flag_) { + auto flag = std::make_shared(std::move(flag_)); + if (flag->handler.arity != ArityAny) + assert(flag->handler.arity == flag->labels.size()); assert(flag->longName != ""); - args.longFlags[flag->longName] = flag; - if (flag->shortName) args.shortFlags[flag->shortName] = flag; + longFlags[flag->longName] = flag; + if (flag->shortName) shortFlags[flag->shortName] = flag; } void Args::parseCmdline(const Strings & _cmdline) @@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; std::vector args; - for (size_t n = 0 ; n < flag.arity; ++n) { + for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.arity == ArityAny) break; - throw UsageError(format("flag '%1%' requires %2% argument(s)") - % name % flag.arity); + if (flag.handler.arity == ArityAny) break; + throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } args.push_back(*pos++); } - flag.handler(std::move(args)); + flag.handler.fun(std::move(args)); return true; }; @@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } -Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { - arity(1); - label("type"); - description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); - handler([ht](std::string s) { - *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); - }); - return *this; + return Flag { + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }} + }; } Strings argvToStrings(int argc, char * * argv) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 967efbe1c..9b5e316a5 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -32,13 +32,59 @@ protected: struct Flag { typedef std::shared_ptr ptr; + + struct Handler + { + std::function)> fun; + size_t arity; + + Handler() {} + + Handler(std::function)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector) { handler(); }) + , arity(0) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function && handler) + : fun([handler{std::move(handler)}](std::vector ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + template + Handler(T * dest) + : fun([=](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + template + Handler(T * dest, const T & val) + : fun([=](std::vector ss) { *dest = val; }) + , arity(0) + { } + }; + std::string longName; char shortName = 0; std::string description; - Strings labels; - size_t arity = 0; - std::function)> handler; std::string category; + Strings labels; + Handler handler; + + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); }; std::map longFlags; @@ -65,49 +111,7 @@ protected: public: - class FlagMaker - { - Args & args; - Flag::ptr flag; - friend class Args; - FlagMaker(Args & args) : args(args), flag(std::make_shared()) { } - public: - ~FlagMaker(); - FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; } - FlagMaker & shortName(char s) { flag->shortName = s; return *this; } - FlagMaker & description(const std::string & s) { flag->description = s; return *this; } - FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; } - FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; } - FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; } - FlagMaker & handler(std::function)> handler) { flag->handler = handler; return *this; } - FlagMaker & handler(std::function handler) { flag->handler = [handler](std::vector) { handler(); }; return *this; } - FlagMaker & handler(std::function handler) { - flag->arity = 1; - flag->handler = [handler](std::vector ss) { handler(std::move(ss[0])); }; - return *this; - } - FlagMaker & category(const std::string & s) { flag->category = s; return *this; } - - template - FlagMaker & dest(T * dest) - { - flag->arity = 1; - flag->handler = [=](std::vector ss) { *dest = ss[0]; }; - return *this; - } - - template - FlagMaker & set(T * dest, const T & val) - { - flag->arity = 0; - flag->handler = [=](std::vector ss) { *dest = val; }; - return *this; - } - - FlagMaker & mkHashTypeFlag(HashType * ht); - }; - - FlagMaker mkFlag(); + void addFlag(Flag && flag); /* Helper functions for constructing flags / positional arguments. */ @@ -116,13 +120,13 @@ public: const std::string & label, const std::string & description, std::function fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({label}) - .description(description) - .arity(1) - .handler([=](std::vector ss) { fun(ss[0]); }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {label}, + .handler = {[=](std::string s) { fun(s); }} + }); } void mkFlag(char shortName, const std::string & name, @@ -135,11 +139,12 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, T * dest, const T & value) { - mkFlag() - .shortName(shortName) - .longName(longName) - .description(description) - .handler([=](std::vector ss) { *dest = value; }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .handler = {[=]() { *dest = value; }} + }); } template @@ -155,18 +160,18 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({"N"}) - .description(description) - .arity(1) - .handler([=](std::vector ss) { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"N"}, + .handler = {[=](std::string s) { I n; - if (!string2Int(ss[0], n)) + if (!string2Int(s, n)) throw UsageError("flag '--%s' requires a integer argument", longName); fun(n); - }); + }} + }); } /* Expect a string argument. */ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7551d97d1..f03e444ec 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -177,12 +177,13 @@ void BaseSetting::toJSON(JSONPlaceholder & out) template void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .arity(1) - .handler([=](std::vector ss) { overriden = true; set(ss[0]); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s); }}, + }); } template<> void BaseSetting::set(const std::string & str) @@ -227,16 +228,18 @@ template<> std::string BaseSetting::to_string() const template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .handler([=](std::vector ss) { override(true); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description(description) - .handler([=](std::vector ss) { override(false); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .handler = {[=]() { override(true); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = description, + .category = category, + .handler = {[=]() { override(false); }} + }); } template<> void BaseSetting::set(const std::string & str) diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ed02227db..00da01f7e 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -14,12 +14,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { expectArg("path", &path); - mkFlag() - .longName("name") - .shortName('n') - .description("name component of the store path") - .labels({"name"}) - .dest(&namePart); + addFlag({ + .longName = "name", + .shortName = 'n', + .description = "name component of the store path", + .labels = {"name"}, + .handler = {&namePart}, + }); } std::string description() override diff --git a/src/nix/build.cc b/src/nix/build.cc index 0b0762836..850e09ce8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,17 +11,19 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile CmdBuild() { - mkFlag() - .longName("out-link") - .shortName('o') - .description("path of the symlink to the build result") - .labels({"path"}) - .dest(&outLink); + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + }); - mkFlag() - .longName("no-link") - .description("do not create a symlink to the build result") - .set(&outLink, Path("")); + addFlag({ + .longName = "no-link", + .description = "do not create a symlink to the build result", + .handler = {&outLink, Path("")}, + }); } std::string description() override diff --git a/src/nix/command.cc b/src/nix/command.cc index 99b24d2a2..71b027719 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -35,16 +35,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { if (recursive) - mkFlag() - .longName("no-recursive") - .description("apply operation to specified paths only") - .set(&this->recursive, false); + addFlag({ + .longName = "no-recursive", + .description = "apply operation to specified paths only", + .handler = {&this->recursive, false}, + }); else - mkFlag() - .longName("recursive") - .shortName('r') - .description("apply operation to closure of the specified paths") - .set(&this->recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "apply operation to closure of the specified paths", + .handler = {&this->recursive, true}, + }); mkFlag(0, "all", "apply operation to the entire store", &all); } @@ -101,11 +103,12 @@ Strings editorFor(const Pos & pos) MixProfile::MixProfile() { - mkFlag() - .longName("profile") - .description("profile to update") - .labels({"path"}) - .dest(&profile); + addFlag({ + .longName = "profile", + .description = "profile to update", + .labels = {"path"}, + .handler = {&profile}, + }); } void MixProfile::updateProfile(const StorePath & storePath) @@ -145,28 +148,30 @@ MixDefaultProfile::MixDefaultProfile() profile = getDefaultProfile(); } -MixEnvironment::MixEnvironment() : ignoreEnvironment(false) { - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) +{ + addFlag({ + .longName = "ignore-environment", + .shortName = 'i', + .description = "clear the entire environment (except those specified with --keep)", + .handler = {&ignoreEnvironment, true}, + }); - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { keep.insert(ss.front()); }); + addFlag({ + .longName = "keep", + .shortName = 'k', + .description = "keep specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { keep.insert(s); }}, + }); - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector ss) { unset.insert(ss.front()); }); + addFlag({ + .longName = "unset", + .shortName = 'u', + .description = "unset specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { unset.insert(s); }}, + }); } void MixEnvironment::setEnviron() { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 85c777d38..77673a1c2 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -19,27 +19,32 @@ struct CmdCopy : StorePathsCommand CmdCopy() : StorePathsCommand(true) { - mkFlag() - .longName("from") - .labels({"store-uri"}) - .description("URI of the source Nix store") - .dest(&srcUri); - mkFlag() - .longName("to") - .labels({"store-uri"}) - .description("URI of the destination Nix store") - .dest(&dstUri); + addFlag({ + .longName = "from", + .description = "URI of the source Nix store", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); - mkFlag() - .longName("no-check-sigs") - .description("do not require that paths are signed by trusted keys") - .set(&checkSigs, NoCheckSigs); + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); - mkFlag() - .longName("substitute-on-destination") - .shortName('s') - .description("whether to try substitutes on the destination store (only supported by SSH)") - .set(&substitute, Substitute); + addFlag({ + .longName = "no-check-sigs", + .description = "do not require that paths are signed by trusted keys", + .handler = {&checkSigs, NoCheckSigs}, + }); + + addFlag({ + .longName = "substitute-on-destination", + .shortName = 's', + .description = "whether to try substitutes on the destination store (only supported by SSH)", + .handler = {&substitute, Substitute}, + }); } std::string description() override diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc index 9c45a935d..2bdf59839 100644 --- a/src/nix/dev-shell.cc +++ b/src/nix/dev-shell.cc @@ -237,16 +237,16 @@ struct CmdDevShell : Common, MixEnvironment CmdDevShell() { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed insted of an interactive shell") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector ss) { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed insted of an interactive shell", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }); + }} + }); } std::string description() override diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 01628cf6c..9b9509d3a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -23,9 +23,7 @@ struct CmdHash : Command mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") @@ -76,9 +74,7 @@ struct CmdToBase : Command CmdToBase(Base base) : base(base) { - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); expectArgs("strings", &args); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1d70ad3d5..937d69206 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -15,12 +15,13 @@ namespace nix { SourceExprCommand::SourceExprCommand() { - mkFlag() - .shortName('f') - .longName("file") - .label("file") - .description("evaluate FILE rather than the default") - .dest(&file); + addFlag({ + .longName = "file", + .shortName = 'f', + .description = "evaluate FILE rather than the default", + .labels = {"file"}, + .handler = {&file} + }); } Value * SourceExprCommand::getSourceExpr(EvalState & state) diff --git a/src/nix/main.cc b/src/nix/main.cc index 2c64c7476..57b8bed9f 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,15 +59,16 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { showHelpAndExit(); }); + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { showHelpAndExit(); }}, + }); - mkFlag() - .longName("help-config") - .description("show configuration options") - .handler([&]() { + addFlag({ + .longName = "help-config", + .description = "show configuration options", + .handler = {[&]() { std::cout << "The following configuration options are available:\n\n"; Table2 tbl; std::map settings; @@ -76,28 +77,33 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs tbl.emplace_back(s.first, s.second.description); printTable(std::cout, tbl); throw Exit(); - }); + }}, + }); - mkFlag() - .longName("print-build-logs") - .shortName('L') - .description("print full build logs on stderr") - .set(&printBuildLogs, true); + addFlag({ + .longName = "print-build-logs", + .shortName = 'L', + .description = "print full build logs on stderr", + .handler = {&printBuildLogs, true}, + }); - mkFlag() - .longName("version") - .description("show version information") - .handler([&]() { printVersion(programName); }); + addFlag({ + .longName = "version", + .description = "show version information", + .handler = {[&]() { printVersion(programName); }}, + }); - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([&]() { useNet = false; }); + addFlag({ + .longName = "no-net", + .description = "disable substituters and consider all previously downloaded files up-to-date", + .handler = {[&]() { useNet = false; }}, + }); - mkFlag() - .longName("refresh") - .description("consider all previously downloaded files out-of-date") - .handler([&]() { refresh = true; }); + addFlag({ + .longName = "refresh", + .description = "consider all previously downloaded files out-of-date", + .handler = {[&]() { refresh = true; }}, + }); } void printFlags(std::ostream & out) override diff --git a/src/nix/run.cc b/src/nix/run.cc index ebfec36d9..b888281a5 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -63,16 +63,16 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment CmdShell() { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed; defaults to '$SHELL'") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector ss) { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed; defaults to '$SHELL'", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }); + }} + }); } std::string description() override diff --git a/src/nix/search.cc b/src/nix/search.cc index 769274543..fcad6be84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -40,16 +40,18 @@ struct CmdSearch : SourceExprCommand, MixJSON { expectArgs("regex", &res); - mkFlag() - .longName("update-cache") - .shortName('u') - .description("update the package search cache") - .handler([&]() { writeCache = true; useCache = false; }); + addFlag({ + .longName = "update-cache", + .shortName = 'u', + .description = "update the package search cache", + .handler = {[&]() { writeCache = true; useCache = false; }} + }); - mkFlag() - .longName("no-cache") - .description("do not use or update the package search cache") - .handler([&]() { writeCache = false; useCache = false; }); + addFlag({ + .longName = "no-cache", + .description = "do not use or update the package search cache", + .handler = {[&]() { writeCache = false; useCache = false; }} + }); } std::string description() override diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 0ede7b468..b6f24599f 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -15,11 +15,12 @@ struct CmdShowDerivation : InstallablesCommand CmdShowDerivation() { - mkFlag() - .longName("recursive") - .shortName('r') - .description("include the dependencies of the specified derivations") - .set(&recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "include the dependencies of the specified derivations", + .handler = {&recursive, true} + }); } std::string description() override diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 5f07448e0..a91465c2a 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand CmdCopySigs() { - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + }); } std::string description() override @@ -98,12 +98,13 @@ struct CmdSignPaths : StorePathsCommand CmdSignPaths() { - mkFlag() - .shortName('k') - .longName("key-file") - .label("file") - .description("file containing the secret signing key") - .dest(&secretKeyFile); + addFlag({ + .longName = "key-file", + .shortName = 'k', + .description = "file containing the secret signing key", + .labels = {"file"}, + .handler = {&secretKeyFile} + }); } std::string description() override diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 4fcc6a721..32efcc3a7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -16,18 +16,20 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand CmdUpgradeNix() { - mkFlag() - .longName("profile") - .shortName('p') - .labels({"profile-dir"}) - .description("the Nix profile to upgrade") - .dest(&profileDir); + addFlag({ + .longName = "profile", + .shortName = 'p', + .description = "the Nix profile to upgrade", + .labels = {"profile-dir"}, + .handler = {&profileDir} + }); - mkFlag() - .longName("nix-store-paths-url") - .labels({"url"}) - .description("URL of the file that contains the store paths of the latest Nix release") - .dest(&storePathsUrl); + addFlag({ + .longName = "nix-store-paths-url", + .description = "URL of the file that contains the store paths of the latest Nix release", + .labels = {"url"}, + .handler = {&storePathsUrl} + }); } std::string description() override diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b0658803..08a36ac50 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,13 +20,13 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }} + }); mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index f9acc7f13..36a3ee863 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -37,11 +37,12 @@ struct CmdWhyDepends : SourceExprCommand expectArg("package", &_package); expectArg("dependency", &_dependency); - mkFlag() - .longName("all") - .shortName('a') - .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path") - .set(&all, true); + addFlag({ + .longName = "all", + .shortName = 'a', + .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .handler = {&all, true}, + }); } std::string description() override From afaa541013549c254f9c54e0865dc254973eea96 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 14:44:00 -0600 Subject: [PATCH 197/350] affinity operator<< --- src/libutil/affinity.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index e3cf33b58..ac2295e4a 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -12,6 +12,17 @@ namespace nix { #if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; + +std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset) +{ + auto count = CPU_COUNT(&cset); + for (int i=0; i < count; ++i) + { + os << (CPU_ISSET(i,&cset) ? "1" : "0"); + } + + return os; +} #endif @@ -47,7 +58,11 @@ void restoreAffinity() #if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - _printError("failed to restore affinity"); + { + std::ostringstream oss; + oss << savedAffinity; + printError("failed to restore CPU affinity %1%", oss.str()); + } #endif } From 8c8f2b74ec3fe37f8661c8411749965b6e8d44d2 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 14:44:42 -0600 Subject: [PATCH 198/350] log as warning --- src/nix-store/nix-store.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 2e2276b4a..57063d42f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -841,8 +841,7 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & p : willSubstitute) subs.emplace_back(p.clone()); store->buildPaths(subs); } catch (Error & e) { - // logWarning(e.info()) TODO: - _printError("warning: %1%", e.msg()); + logWarning(e.info()); } } From e2fc575c6122b40f5d660ae04f269e118354c101 Mon Sep 17 00:00:00 2001 From: Jude Taylor Date: Mon, 4 May 2020 14:42:06 -0700 Subject: [PATCH 199/350] nix auto-gc: use fragment size --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6bab1e37c..d210defe5 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -889,7 +889,7 @@ void LocalStore::autoGC(bool sync) if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); - return (uint64_t) st.f_bavail * st.f_bsize; + return (uint64_t) st.f_bavail * st.f_frsize; }; std::shared_future future; From f30de61578edd9c19744256985185045e7baab84 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 16:19:20 -0600 Subject: [PATCH 200/350] add normaltxt, yellowify->yellowtxt --- src/libutil/fmt.hh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index d182adc3a..bfacfeb4d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -73,18 +73,31 @@ inline std::string fmt(const std::string & fs, const Args & ... args) // are always in yellow. template -struct yellowify +struct yellowtxt { - yellowify(T &s) : value(s) {} + yellowtxt(T &s) : value(s) {} T &value; }; template -std::ostream& operator<<(std::ostream &out, const yellowify &y) +std::ostream& operator<<(std::ostream &out, const yellowtxt &y) { return out << ANSI_YELLOW << y.value << ANSI_NORMAL; } +template +struct normaltxt +{ + normaltxt(T &s) : value(s) {} + T &value; +}; + +template +std::ostream& operator<<(std::ostream &out, const normaltxt &y) +{ + return out << ANSI_NORMAL << y.value; +} + class hintformat { public: @@ -100,7 +113,7 @@ public: template hintformat& operator%(const T &value) { - fmt % yellowify(value); + fmt % yellowtxt(value); return *this; } From 7ffb5efdbc943851d2ee9d0573dca3e96b9bd742 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 4 May 2020 16:19:57 -0600 Subject: [PATCH 201/350] appending to hints; remove _printError --- src/error-demo/error-demo.cc | 7 +++++-- src/libstore/build.cc | 12 ++++++++---- src/libutil/logging.hh | 1 - src/nix-daemon/nix-daemon.cc | 7 +++++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 514258416..82b03d71b 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -26,10 +26,13 @@ int main() // ErrorInfo constructor try { - auto e = Error("generic error"); + auto e = Error("some error"); throw DemoError(e.info()); } catch (Error &e) { - logger->logEI(e.info()); + ErrorInfo ei = e.info(); + string prevhint = (e.info().hint.has_value() ? e.info().hint->str() : ""); + ei.hint = std::optional(hintfmt("previous hint was: %s", normaltxt(prevhint))); + logger->logEI(ei); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bacbd5808..f8cc1ce36 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -488,14 +488,18 @@ void handleDiffHook( auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first)); + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); } catch (Error & error) { - // logError(error.info()) - // TODO append message onto errorinfo... - _printError("diff hook execution failed: %s", error.what()); + ErrorInfo ei = error.info(); + string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : ""); + ei.hint = std::optional(hintfmt("diff hook execution failed: %s", prevhint)); + logError(ei); } } } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f03cdf41..0c4980b83 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -149,7 +149,6 @@ extern Verbosity verbosity; /* suppress msgs > this */ } \ } while (0) -#define _printError(args...) printMsg(lvlError, args) #define printError(args...) printMsg(lvlError, args) #define printInfo(args...) printMsg(lvlInfo, args) #define printTalkative(args...) printMsg(lvlTalkative, args) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 3cd2c3a6b..5336db136 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -246,9 +246,12 @@ static void daemonLoop(char * * argv) } catch (Interrupted & e) { return; - } catch (Error & e) { + } catch (Error & error) { // TODO append error message - _printError("error processing connection: %1%", e.msg()); + ErrorInfo ei = error.info(); + string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : ""); + ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint)); + logError(ei); } } } From 04967dee9d08befe4661e6fa5da4a00da0538a13 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 5 May 2020 13:04:36 +0300 Subject: [PATCH 202/350] Wait for build users when none are available --- src/libstore/build.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 147093fae..1b7e6d75e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -507,6 +507,9 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; + bool findFreeUser(); + + string user; uid_t uid; gid_t gid; @@ -526,10 +529,19 @@ public: }; - UserLock::UserLock() { assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); + + if (findFreeUser()) return; + + printError("waiting for build users"); + + do std::this_thread::sleep_for(std::chrono::seconds(2)); while (! findFreeUser()); +} + +bool UserLock::findFreeUser() { /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -559,7 +571,6 @@ UserLock::UserLock() throw Error(format("the user '%1%' in the group '%2%' does not exist") % i % settings.buildUsersGroup); - createDirs(settings.nixStateDir + "/userpool"); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); @@ -590,16 +601,12 @@ UserLock::UserLock() supplementaryGIDs.resize(ngroups); #endif - return; + return true; } } - - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the '%1%' group") - % settings.buildUsersGroup); + return false; } - void UserLock::kill() { killUser(uid); From f132d82a796c91fcb741c127f37c963622b4cae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2020 15:18:23 +0200 Subject: [PATCH 203/350] nix --help: Group commands --- src/libutil/ansicolor.hh | 24 ++++++------ src/libutil/args.cc | 57 ++++++++++++++++------------- src/libutil/args.hh | 18 +++++---- src/nix/add-to-store.cc | 2 + src/nix/cat.cc | 8 +++- src/nix/command.hh | 4 ++ src/nix/copy.cc | 2 + src/nix/dev-shell.cc | 2 + src/nix/doctor.cc | 2 + src/nix/dump-path.cc | 2 + src/nix/edit.cc | 2 + src/nix/eval.cc | 2 + src/nix/hash.cc | 4 ++ src/nix/log.cc | 2 + src/nix/ls.cc | 8 +++- src/nix/main.cc | 18 ++++++--- src/nix/make-content-addressable.cc | 3 ++ src/nix/optimise-store.cc | 2 + src/nix/path-info.cc | 2 + src/nix/ping-store.cc | 2 + src/nix/show-config.cc | 2 + src/nix/show-derivation.cc | 2 + src/nix/sigs.cc | 4 ++ src/nix/upgrade-nix.cc | 2 + src/nix/verify.cc | 2 + src/nix/why-depends.cc | 2 + 26 files changed, 125 insertions(+), 55 deletions(-) diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 390bd4d17..8ae07b092 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,13 +1,15 @@ -#pragma once +#pragma once + +namespace nix { + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_ITALIC "\e[3m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_YELLOW "\e[33;1m" +#define ANSI_BLUE "\e[34;1m" -namespace nix -{ - /* Some ANSI escape sequences. */ - #define ANSI_NORMAL "\e[0m" - #define ANSI_BOLD "\e[1m" - #define ANSI_FAINT "\e[2m" - #define ANSI_RED "\e[31;1m" - #define ANSI_GREEN "\e[32;1m" - #define ANSI_YELLOW "\e[33;1m" - #define ANSI_BLUE "\e[34;1m" } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 91fc2f581..f829415d1 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -59,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline) void Args::printHelp(const string & programName, std::ostream & out) { - std::cout << "Usage: " << programName << " ..."; + std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 @@ -70,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out) auto s = description(); if (s != "") - std::cout << "\nSummary: " << s << ".\n"; + std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; if (longFlags.size()) { std::cout << "\n"; - std::cout << "Flags:\n"; + std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; printFlags(out); } } @@ -181,7 +181,7 @@ std::string renderLabels(const Strings & labels) std::string res; for (auto label : labels) { for (auto & c : label) c = std::toupper(c); - res += " <" + label + ">"; + res += " " ANSI_ITALIC + label + ANSI_NORMAL; } return res; } @@ -190,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table) { size_t max = 0; for (auto & row : table) - max = std::max(max, row.first.size()); + max = std::max(max, filterANSIEscapes(row.first, true).size()); for (auto & row : table) { out << " " << row.first - << std::string(max - row.first.size() + 2, ' ') + << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') << row.second << "\n"; } } @@ -204,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out) auto exs = examples(); if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; + out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; for (auto & ex : exs) out << "\n" << " " << ex.description << "\n" // FIXME: wrap @@ -221,49 +220,55 @@ MultiCommand::MultiCommand(const Commands & commands) auto i = commands.find(ss[0]); if (i == commands.end()) throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second(); - command->_name = ss[0]; + command = {ss[0], i->second()}; }}); + + categories[Command::catDefault] = "Available commands"; } void MultiCommand::printHelp(const string & programName, std::ostream & out) { if (command) { - command->printHelp(programName + " " + command->name(), out); + command->second->printHelp(programName + " " + command->first, out); return; } - out << "Usage: " << programName << " ... ...\n"; + out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); - out << "\n"; - out << "Common flags:\n"; + out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; printFlags(out); - out << "\n"; - out << "Available commands:\n"; + std::map>> commandsByCategory; - Table2 table; - for (auto & i : commands) { - auto command = i.second(); - command->_name = i.first; - auto descr = command->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command->name(), descr)); + for (auto & [name, commandFun] : commands) { + auto command = commandFun(); + commandsByCategory[command->category()].insert_or_assign(name, command); + } + + for (auto & [category, commands] : commandsByCategory) { + out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); + + Table2 table; + for (auto & [name, command] : commands) { + auto descr = command->description(); + if (!descr.empty()) + table.push_back(std::make_pair(name, descr)); + } + printTable(out, table); } - printTable(out, table); } bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; + if (command && command->second->processFlag(pos, end)) return true; return false; } bool MultiCommand::processArgs(const Strings & args, bool finish) { if (command) - return command->processArgs(args, finish); + return command->second->processArgs(args, finish); else return Args::processArgs(args, finish); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 9b5e316a5..1932e6a8a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -197,17 +197,10 @@ public: run() method. */ struct Command : virtual Args { -private: - std::string _name; - friend class MultiCommand; -public: - virtual ~Command() { } - std::string name() { return _name; } - virtual void prepare() { }; virtual void run() = 0; @@ -221,6 +214,12 @@ public: virtual Examples examples() { return Examples(); } + typedef int Category; + + static constexpr Category catDefault = 0; + + virtual Category category() { return catDefault; } + void printHelp(const string & programName, std::ostream & out) override; }; @@ -233,7 +232,10 @@ class MultiCommand : virtual Args public: Commands commands; - std::shared_ptr command; + std::map categories; + + // Selected command, if any. + std::optional>> command; MultiCommand(const Commands & commands); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 00da01f7e..1d298903b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -34,6 +34,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { if (!namePart) namePart = baseNameOf(path); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 851f90abd..fd91f2036 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -30,9 +30,11 @@ struct CmdCatStore : StoreCommand, MixCat std::string description() override { - return "print the contents of a store file on stdout"; + return "print the contents of a file in the Nix store on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(store->getFSAccessor()); @@ -51,9 +53,11 @@ struct CmdCatNar : StoreCommand, MixCat std::string description() override { - return "print the contents of a file inside a NAR file"; + return "print the contents of a file inside a NAR file on stdout"; } + Category category() override { return catUtility; } + void run(ref store) override { cat(makeNarAccessor(make_ref(readFile(narPath)))); diff --git a/src/nix/command.hh b/src/nix/command.hh index bf43d950f..959d5f19d 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -10,6 +10,10 @@ namespace nix { extern std::string programPath; +static constexpr Command::Category catSecondary = 100; +static constexpr Command::Category catUtility = 101; +static constexpr Command::Category catNixInstallation = 102; + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 77673a1c2..c7c38709d 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -80,6 +80,8 @@ struct CmdCopy : StorePathsCommand }; } + Category category() override { return catSecondary; } + ref createStore() override { return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc index 2bdf59839..d300f6a23 100644 --- a/src/nix/dev-shell.cc +++ b/src/nix/dev-shell.cc @@ -328,6 +328,8 @@ struct CmdPrintDevEnv : Common }; } + Category category() override { return catUtility; } + void run(ref store) override { auto buildEnvironment = getBuildEnvironment(store).first; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 0aa634d6e..cc36354f1 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -43,6 +43,8 @@ struct CmdDoctor : StoreCommand return "check your system for potential problems and print a PASS or FAIL for each check."; } + Category category() override { return catNixInstallation; } + void run(ref store) override { logger->log("Running checks against store uri: " + store->getUri()); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index bb741b572..e1de71bf8 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,6 +20,8 @@ struct CmdDumpPath : StorePathCommand }; } + Category category() override { return catUtility; } + void run(ref store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 1683eada0..067d3a973 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -25,6 +25,8 @@ struct CmdEdit : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { auto state = getEvalState(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 86a1e8b68..26e98ac2a 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -45,6 +45,8 @@ struct CmdEval : MixJSON, InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { if (raw && json) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9b9509d3a..366314227 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -41,6 +41,8 @@ struct CmdHash : Command : "print cryptographic hash of the NAR serialisation of a path"; } + Category category() override { return catUtility; } + void run() override { for (auto path : paths) { @@ -87,6 +89,8 @@ struct CmdToBase : Command "SRI"); } + Category category() override { return catUtility; } + void run() override { for (auto s : args) diff --git a/src/nix/log.cc b/src/nix/log.cc index 795991cb7..3fe22f6c2 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -31,6 +31,8 @@ struct CmdLog : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { settings.readOnlyMode = true; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 8590199d7..6ae4df27e 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -100,9 +100,11 @@ struct CmdLsStore : StoreCommand, MixLs std::string description() override { - return "show information about a store path"; + return "show information about a path in the Nix store"; } + Category category() override { return catUtility; } + void run(ref store) override { list(store->getFSAccessor()); @@ -131,9 +133,11 @@ struct CmdLsNar : Command, MixLs std::string description() override { - return "show information about the contents of a NAR file"; + return "show information about a path inside a NAR file"; } + Category category() override { return catUtility; } + void run() override { list(makeNarAccessor(make_ref(readFile(narPath, true)))); diff --git a/src/nix/main.cc b/src/nix/main.cc index 57b8bed9f..5cf09c4f0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -59,6 +59,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { + categories.clear(); + categories[Command::catDefault] = "Main commands"; + categories[catSecondary] = "Infrequently used commands"; + categories[catUtility] = "Utility/scripting commands"; + categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation"; + addFlag({ .longName = "help", .description = "show usage information", @@ -111,8 +117,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs Args::printFlags(out); std::cout << "\n" - "In addition, most configuration settings can be overriden using '-- '.\n" - "Boolean settings can be overriden using '--' or '--no-'. See 'nix\n" + "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" + "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" "--help-config' for a list of configuration settings.\n"; } @@ -121,10 +127,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs MultiCommand::printHelp(programName, out); #if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-'.\n"; + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; #endif - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; } void showHelpAndExit() @@ -191,8 +197,8 @@ void mainWrapped(int argc, char * * argv) if (args.refresh) settings.tarballTtl = 0; - args.command->prepare(); - args.command->run(); + args.command->second->prepare(); + args.command->second->run(); } } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f9c7fef3f..8803461f4 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -31,6 +31,9 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }, }; } + + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { auto paths = store->topoSortPaths(storePathsToSet(storePaths)); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index fed012b04..b45951879 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -23,6 +23,8 @@ struct CmdOptimiseStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { store->optimiseStore(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 45ec297d2..88d7fffd4 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -29,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } + Category category() override { return catSecondary; } + Examples examples() override { return { diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3a2e542a3..127397a29 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -21,6 +21,8 @@ struct CmdPingStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { store->connect(); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 6104b10bc..4fd8886de 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -13,6 +13,8 @@ struct CmdShowConfig : Command, MixJSON return "show the Nix configuration"; } + Category category() override { return catUtility; } + void run() override { if (json) { diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index b6f24599f..22c569f3c 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -42,6 +42,8 @@ struct CmdShowDerivation : InstallablesCommand }; } + Category category() override { return catUtility; } + void run(ref store) override { auto drvPaths = toDerivations(store, installables, true); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a91465c2a..6c9b9a792 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,6 +27,8 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -112,6 +114,8 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } + Category category() override { return catUtility; } + void run(ref store, StorePaths storePaths) override { if (secretKeyFile.empty()) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 32efcc3a7..678780f33 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -51,6 +51,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }; } + Category category() override { return catNixInstallation; } + void run(ref store) override { evalSettings.pureEval = true; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 08a36ac50..cf1fa6a99 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -49,6 +49,8 @@ struct CmdVerify : StorePathsCommand }; } + Category category() override { return catSecondary; } + void run(ref store, StorePaths storePaths) override { std::vector> substituters; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 36a3ee863..6057beedb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -68,6 +68,8 @@ struct CmdWhyDepends : SourceExprCommand }; } + Category category() override { return catSecondary; } + void run(ref store) override { auto package = parseInstallable(*this, store, _package, false); From 909b4a882057a92ae8b40b8afdab3c4ac30da915 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 May 2020 15:27:47 +0200 Subject: [PATCH 204/350] nix doctor: Consistency --- src/nix/doctor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index cc36354f1..82e92cdd0 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -40,7 +40,7 @@ struct CmdDoctor : StoreCommand std::string description() override { - return "check your system for potential problems and print a PASS or FAIL for each check."; + return "check your system for potential problems and print a PASS or FAIL for each check"; } Category category() override { return catNixInstallation; } From fd4911269f9a9ce0f9647c5fb88658074f654eff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 May 2020 10:54:01 +0200 Subject: [PATCH 205/350] Revert "Merge pull request #3558 from LnL7/ssh-ng-stderr" This reverts commit 3ebfbecdd187002569257f7cb183bf9e0b39af1e, reversing changes made to c089c52d5f1cff888552f485775b74226dcbe618. https://github.com/NixOS/nix/pull/3558 --- src/libstore/remote-store.cc | 6 ++---- src/libutil/logging.cc | 10 ---------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 07ef79382..8c55da268 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -779,10 +779,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return std::make_exception_ptr(Error(status, error)); } - else if (msg == STDERR_NEXT) { - string s = chomp(readString(from)); - printMsg(lvlVomit, "stderr %s", s); - } + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); else if (msg == STDERR_START_ACTIVITY) { auto act = readNum(from); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 777650de5..3cc4ef8f1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -63,16 +63,6 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } - void result(ActivityId act, ResultType type, const std::vector & fields) override - { - if (type == resBuildLogLine || type == resPostBuildLogLine) { - assert(0 < fields.size()); - assert(fields[0].type == Logger::Field::tString); - auto lastLine = fields[0].s; - log(lvlInfo, lastLine); - } - } - void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override From 9be46859a90d5a0771e079615d324644a08efb41 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Wed, 6 May 2020 11:21:12 +0200 Subject: [PATCH 206/350] libstore/build.cc: more explicit about form of output Be more explicit about why we expect a regular file as output when outputHashMode=flat for a fixed output derivation. --- src/libstore/build.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 572634765..e0c7324c7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3678,7 +3678,8 @@ void DerivationGoal::registerOutputs() /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); + format("output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat).") % path); } /* Check the hash. In hash mode, move the path produced by From 85c1932c9425c15d8d16da4777ed27667874a744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 5 May 2020 12:09:46 +0100 Subject: [PATCH 207/350] nix/search: no error for empty search results if json is enabled - result list will be always empty if --json is passed - for scripts an empty search result is not really an error, we rather want to distinguish between evaluation errors and empty results --- src/nix/search.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index fcad6be84..ba72c1e79 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -265,7 +265,7 @@ struct CmdSearch : SourceExprCommand, MixJSON throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); } - if (results.size() == 0) + if (!json && results.size() == 0) throw Error("no results for the given search term(s)!"); RunPager pager; From 58ed1e6d6842688ac2d32355265e0d89049f7e36 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Tue, 5 May 2020 17:56:46 +0200 Subject: [PATCH 208/350] WIP: add unit tests for libutil This is a proof on concept to evaluate writing unit tests for Nix using google test (https://github.com/google/googletest). In order to execute tests: $ make unit-tests $ ./unit-tests The Makefile rules for `unit-tests` is a complete hack. --- Makefile | 3 +- release-common.nix | 1 + tests/unit-tests/local.mk | 4 + tests/unit-tests/test-util.cc | 594 ++++++++++++++++++++++++++++++++++ 4 files changed, 601 insertions(+), 1 deletion(-) create mode 100644 tests/unit-tests/local.mk create mode 100644 tests/unit-tests/test-util.cc diff --git a/Makefile b/Makefile index e3057c36c..06de35a27 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,8 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk + tests/plugins/local.mk \ + tests/unit-tests/local.mk -include Makefile.config diff --git a/release-common.nix b/release-common.nix index b6c8f3d81..7e7de005d 100644 --- a/release-common.nix +++ b/release-common.nix @@ -55,6 +55,7 @@ rec { # Tests git mercurial + gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk new file mode 100644 index 000000000..ed11261c1 --- /dev/null +++ b/tests/unit-tests/local.mk @@ -0,0 +1,4 @@ +LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec +unit-tests: + echo $(nix_LDFLAGS) + $(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc new file mode 100644 index 000000000..05d8a0039 --- /dev/null +++ b/tests/unit-tests/test-util.cc @@ -0,0 +1,594 @@ +#include "../../src/libutil/util.hh" +#include "../../src/libutil/types.hh" + +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + + TEST(absPath, doesntChangeRoot) { + auto p = absPath("/"); + + ASSERT_STREQ(p.c_str(), "/"); + } + + TEST(absPath, turnsEmptyPathIntoCWD) { + char cwd[PATH_MAX+1]; + auto p = absPath(""); + + ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX)); + } + + TEST(absPath, usesOptionalBasePathWhenGiven) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_STREQ(p.c_str(), cwd); + } + + TEST(absPath, isIdempotent) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_STREQ(p1.c_str(), p2.c_str()); + } + + + TEST(absPath, pathIsCanonicalised) { + auto path = "/some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot"); + ASSERT_STREQ(p1.c_str(), p2.c_str()); + } + + /* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + + TEST(canonPath, removesTrailingSlashes) { + auto path = "/this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, removesDots) { + auto path = "/this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, removesDots2) { + auto path = "/this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + } + + TEST(canonPath, requiresAbsolutePath) { + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); + } + + /* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + + // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo") + // should both return "" but it actually returns "/" in both cases + TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { + auto p = dirOf("/"); + + ASSERT_STREQ(p.c_str(), ""); + } + + TEST(dirOf, returnsFirstPathComponent) { + auto p1 = dirOf("/dir/"); + ASSERT_STREQ(p1.c_str(), "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_STREQ(p2.c_str(), "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_STREQ(p3.c_str(), "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_STREQ(p4.c_str(), "/dir/.."); + } + + /* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + + TEST(baseNameOf, emptyPath) { + auto p1 = baseNameOf(""); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + TEST(baseNameOf, pathOnRoot) { + auto p1 = baseNameOf("/dir"); + ASSERT_STREQ(std::string(p1).c_str(), "dir"); + } + + TEST(baseNameOf, relativePath) { + auto p1 = baseNameOf("dir/foo"); + ASSERT_STREQ(std::string(p1).c_str(), "foo"); + } + + TEST(baseNameOf, pathWithTrailingSlashRoot) { + auto p1 = baseNameOf("/"); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return + // "" but it actually returns "dir" + TEST(baseNameOf, DISABLED_trailingSlash) { + auto p1 = baseNameOf("/dir/"); + ASSERT_STREQ(std::string(p1).c_str(), ""); + } + + /* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + + TEST(isInDir, trivialCase) { + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); + } + + TEST(isInDir, notInDir) { + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); + } + + // XXX: hm, bug or feature? :) Looking at the implementation + // this might be problematic. + TEST(isInDir, emptyDir) { + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); + } + + /* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + + TEST(isDirOrInDir, trueForSameDirectory) { + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); + } + + TEST(isDirOrInDir, trueForEmptyPaths) { + ASSERT_EQ(isDirOrInDir("", ""), true); + } + + TEST(isDirOrInDir, falseForDisjunctPaths) { + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); + } + + TEST(isDirOrInDir, relativePaths) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); + } + + // XXX: while it is possible to use "." or ".." in the + // first argument this doesn't seem to work in the second. + TEST(isDirOrInDir, DISABLED_shouldWork) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); + + } + + /* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + + TEST(pathExists, rootExists) { + ASSERT_TRUE(pathExists("/")); + } + + TEST(pathExists, cwdExists) { + ASSERT_TRUE(pathExists(".")); + } + + TEST(pathExists, bogusPathDoesNotExist) { + ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + } + + /* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + + TEST(concatStringsSep, buildCommaSeparatedString) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + } + + TEST(concatStringsSep, buildStringWithEmptySeparator) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + } + + TEST(concatStringsSep, buildSingleString) { + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); + } + + /* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + + TEST(hasPrefix, emptyStringHasNoPrefix) { + ASSERT_FALSE(hasPrefix("", "foo")); + } + + TEST(hasPrefix, emptyStringIsAlwaysPrefix) { + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); + } + + TEST(hasPrefix, trivialCase) { + ASSERT_TRUE(hasPrefix("foobar", "foo")); + } + + /* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + + TEST(hasSuffix, emptyStringHasNoSuffix) { + ASSERT_FALSE(hasSuffix("", "foo")); + } + + TEST(hasSuffix, trivialCase) { + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); + } + + /* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + + TEST(base64Encode, emptyString) { + ASSERT_STREQ(base64Encode("").c_str(), ""); + } + + TEST(base64Encode, encodesAString) { + ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + } + + TEST(base64Encode, encodeAndDecode) { + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_STREQ(decoded.c_str(), s); + } + + /* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + + TEST(base64Decode, emptyString) { + ASSERT_STREQ(base64Decode("").c_str(), ""); + } + + TEST(base64Decode, decodeAString) { + ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum"); + } + + /* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + + TEST(toLower, emptyString) { + ASSERT_STREQ(toLower("").c_str(), ""); + } + + TEST(toLower, nonLetters) { + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_STREQ(toLower(s).c_str(), s); + } + + // XXX: std::tolower() doesn't cover this. This probably doesn't matter + // since the context is paths and store paths specifically where such + // characters are not allowed. + TEST(toLower, DISABLED_umlauts) { + auto s = "ÄÖÜ"; + ASSERT_STREQ(toLower(s).c_str(), "äöü"); + } + + /* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + + TEST(string2Float, emptyString) { + double n; + ASSERT_EQ(string2Float("", n), false); + } + + TEST(string2Float, trivialConversions) { + double n; + ASSERT_EQ(string2Float("1.0", n), true); + ASSERT_EQ(n, 1.0); + + ASSERT_EQ(string2Float("0.0", n), true); + ASSERT_EQ(n, 0.0); + + ASSERT_EQ(string2Float("-100.25", n), true); + ASSERT_EQ(n, (-100.25)); + } + + /* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + + TEST(string2Int, emptyString) { + double n; + ASSERT_EQ(string2Int("", n), false); + } + + TEST(string2Int, trivialConversions) { + double n; + ASSERT_EQ(string2Int("1", n), true); + ASSERT_EQ(n, 1); + + ASSERT_EQ(string2Int("0", n), true); + ASSERT_EQ(n, 0); + + ASSERT_EQ(string2Int("-100", n), true); + ASSERT_EQ(n, (-100)); + } + + /* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + + TEST(statusOk, zeroIsOk) { + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); + } + + /* ---------------------------------------------------------------------------- + * replaceInSet : XXX: this function isn't called anywhere! + * --------------------------------------------------------------------------*/ + + + /* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + + TEST(rewriteStrings, emptyString) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), ""); + } + + TEST(rewriteStrings, emptyRewrites) { + StringMap rewrites; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + } + + TEST(rewriteStrings, successfulRewrite) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that"); + } + + TEST(rewriteStrings, doesntOccur) { + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + + TEST(replaceStrings, emptyString) { + ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), ""); + ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that"); + } + + TEST(replaceStrings, successfulReplace) { + ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that"); + } + + TEST(replaceStrings, doesntOccur) { + ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + + TEST(trim, emptyString) { + ASSERT_STREQ(trim("").c_str(), ""); + } + + TEST(trim, removesWhitespace) { + ASSERT_STREQ(trim("foo").c_str(), "foo"); + ASSERT_STREQ(trim(" foo ").c_str(), "foo"); + ASSERT_STREQ(trim(" foo bar baz").c_str(), "foo bar baz"); + ASSERT_STREQ(trim(" \t foo bar baz\n").c_str(), "foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + + TEST(chomp, emptyString) { + ASSERT_STREQ(chomp("").c_str(), ""); + } + + TEST(chomp, removesWhitespace) { + ASSERT_STREQ(chomp("foo").c_str(), "foo"); + ASSERT_STREQ(chomp("foo ").c_str(), "foo"); + ASSERT_STREQ(chomp(" foo ").c_str(), " foo"); + ASSERT_STREQ(chomp(" foo bar baz ").c_str(), " foo bar baz"); + ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + + TEST(quoteStrings, empty) { + Strings s = { }; + Strings expected = { }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, emptyStrings) { + Strings s = { "", "", "" }; + Strings expected = { "''", "''", "''" }; + ASSERT_EQ(quoteStrings(s), expected); + + } + + TEST(quoteStrings, trivialQuote) { + Strings s = { "foo", "bar", "baz" }; + Strings expected = { "'foo'", "'bar'", "'baz'" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, quotedStrings) { + Strings s = { "'foo'", "'bar'", "'baz'" }; + Strings expected = { "''foo''", "''bar''", "''baz''" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + /* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + + TEST(tokenizeString, empty) { + auto s = ""; + Strings expected = { }; + + ASSERT_EQ(tokenizeString(""), expected); + } + + TEST(tokenizeString, tokenizeSpacesWithDefaults) { + auto s = "foo bar baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsWithDefaults) { + auto s = "foo\tbar\tbaz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { + auto s = "foo\t bar\t baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { + auto s = "foo\t\n bar\t\n baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString(s2), expected2); + } + + TEST(tokenizeString, tokenizeWithCustomSep) { + auto s = "foo\n,bar\n,baz\n"; + Strings expected = { "foo\n", "bar\n", "baz\n" }; + + ASSERT_EQ(tokenizeString(s, ","), expected); + } + + /* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + + TEST(get, emptyContainer) { + StringMap s = { }; + auto expected = std::nullopt; + + ASSERT_EQ(get(s, "one"), expected); + } + + TEST(get, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(get(s, "one"), expected); + } + + /* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + + TEST(filterANSIEscapes, emptyString) { + auto s = ""; + auto expected = ""; + + ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected); + } + + TEST(filterANSIEscapes, doesntChangePrintableChars) { + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_STREQ(filterANSIEscapes(s).c_str(), s); + } + + TEST(filterANSIEscapes, filtersColorCodes) { + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" ); + ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " ); + ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A " ); + ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A B" ); + ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A B C" ); + } + + TEST(filterANSIEscapes, expandsTabs) { + auto s = "foo\tbar\tbaz"; + + ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo bar baz" ); + } + +} From e76ad2e48a71aa9804311f4dfb34374a5295b3b6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 6 May 2020 14:07:20 -0600 Subject: [PATCH 209/350] implement SysError errno handling --- src/error-demo/error-demo.cc | 8 ++++++++ src/libutil/error.cc | 8 -------- src/libutil/error.hh | 13 +++++++------ src/libutil/fmt.hh | 8 ++++---- src/libutil/util.cc | 3 ++- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 82b03d71b..b1b313e32 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -35,6 +35,14 @@ int main() logger->logEI(ei); } + // SysError; picks up errno + try { + auto x = readFile(-1); + } + catch (Error &e) { + std::cout << "error: " << e.sname() << std::endl; + logError(e.info()); + } // For completeness sake, info through vomit levels. // But this is maybe a heavy format for those. diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 91fa1ccd8..d4305ddd8 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -16,14 +16,6 @@ BaseError & BaseError::addPrefix(const FormatOrString & fs) return *this; } - -std::string SysError::addErrno(const std::string & s) -{ - errNo = errno; - return s + ": " + strerror(errNo); -} - - std::optional ErrorInfo::programName = std::nullopt; std::ostream& operator<<(std::ostream &os, const hintformat &hf) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 86cff5609..2155ad344 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -160,13 +160,14 @@ public: template SysError(const Args & ... args) - : Error(args...) // TODO addErrNo for hintfmt - // : Error(addErrno(hintfmt(args...))) - { } + :Error("") + { + errNo = errno; + auto hf = hintfmt(args...); + err.hint = hintfmt("%1% : %2%", normaltxt(hf.str()), strerror(errNo)); + } -private: - - std::string addErrno(const std::string & s); + virtual const char* sname() const override { return "SysError"; } }; } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index bfacfeb4d..12ab9c407 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -75,8 +75,8 @@ inline std::string fmt(const std::string & fs, const Args & ... args) template struct yellowtxt { - yellowtxt(T &s) : value(s) {} - T &value; + yellowtxt(const T &s) : value(s) {} + const T &value; }; template @@ -88,8 +88,8 @@ std::ostream& operator<<(std::ostream &out, const yellowtxt &y) template struct normaltxt { - normaltxt(T &s) : value(s) {} - T &value; + normaltxt(const T &s) : value(s) {} + const T &value; }; template diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 68dc1b738..e8f22ab71 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -312,7 +312,8 @@ string readFile(const Path & path, bool drain) void readFile(const Path & path, Sink & sink) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%s'", path); + if (!fd) + throw SysError("opening file '%s'", path); drainFD(fd.get(), sink); } From e3901638b50d91ddc9c356adc35ded00a00fe94b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 6 May 2020 15:01:13 -0600 Subject: [PATCH 210/350] todo removal --- src/error-demo/error-demo.cc | 1 - src/libstore/gc.cc | 1 - src/nix-daemon/nix-daemon.cc | 1 - src/nix/upgrade-nix.cc | 2 -- 4 files changed, 5 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index b1b313e32..0d3dcf616 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -23,7 +23,6 @@ int main() logger->logEI(e.info()); } - // ErrorInfo constructor try { auto e = Error("some error"); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4f6f78a99..629c4360c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -41,7 +41,6 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - // TODO: info? printError("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5336db136..731ab22e5 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -247,7 +247,6 @@ static void daemonLoop(char * * argv) } catch (Interrupted & e) { return; } catch (Error & error) { - // TODO append error message ErrorInfo ei = error.info(); string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : ""); ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint)); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 8f4e529bc..831a83fd1 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -64,13 +64,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - // TODO change to info? logWarning( ErrorInfo { .name = "Version update", .hint = hintfmt("would upgrade to version %s", version) }); - // printError("would upgrade to version %s", version); return; } From 479e8bf00b680343231d2b618e130141079a2097 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 16:08:15 +0200 Subject: [PATCH 211/350] Manual: Fix typo --- doc/manual/advanced-topics/post-build-hook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 08a7a772f..57ee99e84 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example Now, copy the path back from the cache: -$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... warning: you did not specify '--add-root'; the result might be removed by the garbage collector /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example From 41caaaad365b8e5260069c20458a2043254a6989 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 16:37:33 +0200 Subject: [PATCH 212/350] Manual: Typo --- doc/manual/advanced-topics/post-build-hook.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 57ee99e84..acfe9e3cc 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= -we will restart the Nix daemon a later step. +We will restart the Nix daemon in a later step.
From 987b3d6469f03bff59614bd3b9097df858d07263 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 18:10:07 +0200 Subject: [PATCH 213/350] Use ASSERT_EQ instead of ASSERT_STREQ No need to use `c_str()` in combination with `ASSERT_STREQ`. It's possible to just use ASSERT_EQ on std::string --- tests/unit-tests/test-util.cc | 108 +++++++++++++++++----------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index 05d8a0039..62ca8f11c 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -12,14 +12,14 @@ namespace nix { TEST(absPath, doesntChangeRoot) { auto p = absPath("/"); - ASSERT_STREQ(p.c_str(), "/"); + ASSERT_EQ(p, "/"); } TEST(absPath, turnsEmptyPathIntoCWD) { char cwd[PATH_MAX+1]; auto p = absPath(""); - ASSERT_STREQ(p.c_str(), getcwd((char*)&cwd, PATH_MAX)); + ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); } TEST(absPath, usesOptionalBasePathWhenGiven) { @@ -28,7 +28,7 @@ namespace nix { auto p = absPath("", cwd); - ASSERT_STREQ(p.c_str(), cwd); + ASSERT_EQ(p, cwd); } TEST(absPath, isIdempotent) { @@ -37,7 +37,7 @@ namespace nix { auto p1 = absPath(cwd); auto p2 = absPath(p1); - ASSERT_STREQ(p1.c_str(), p2.c_str()); + ASSERT_EQ(p1, p2); } @@ -46,8 +46,8 @@ namespace nix { auto p1 = absPath(path); auto p2 = absPath(p1); - ASSERT_STREQ(p1.c_str(), "/some/path/with/trailing/dot"); - ASSERT_STREQ(p1.c_str(), p2.c_str()); + ASSERT_EQ(p1, "/some/path/with/trailing/dot"); + ASSERT_EQ(p1, p2); } /* ---------------------------------------------------------------------------- @@ -58,21 +58,21 @@ namespace nix { auto path = "/this/is/a/path//"; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, removesDots) { auto path = "/this/./is/a/path/./"; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, removesDots2) { auto path = "/this/a/../is/a////path/foo/.."; auto p = canonPath(path); - ASSERT_STREQ(p.c_str(), "/this/is/a/path"); + ASSERT_EQ(p, "/this/is/a/path"); } TEST(canonPath, requiresAbsolutePath) { @@ -91,18 +91,18 @@ namespace nix { TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { auto p = dirOf("/"); - ASSERT_STREQ(p.c_str(), ""); + ASSERT_EQ(p, ""); } TEST(dirOf, returnsFirstPathComponent) { auto p1 = dirOf("/dir/"); - ASSERT_STREQ(p1.c_str(), "/dir"); + ASSERT_EQ(p1, "/dir"); auto p2 = dirOf("/dir"); - ASSERT_STREQ(p2.c_str(), "/"); + ASSERT_EQ(p2, "/"); auto p3 = dirOf("/dir/.."); - ASSERT_STREQ(p3.c_str(), "/dir"); + ASSERT_EQ(p3, "/dir"); auto p4 = dirOf("/dir/../"); - ASSERT_STREQ(p4.c_str(), "/dir/.."); + ASSERT_EQ(p4, "/dir/.."); } /* ---------------------------------------------------------------------------- @@ -111,29 +111,29 @@ namespace nix { TEST(baseNameOf, emptyPath) { auto p1 = baseNameOf(""); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } TEST(baseNameOf, pathOnRoot) { auto p1 = baseNameOf("/dir"); - ASSERT_STREQ(std::string(p1).c_str(), "dir"); + ASSERT_EQ(std::string(p1), "dir"); } TEST(baseNameOf, relativePath) { auto p1 = baseNameOf("dir/foo"); - ASSERT_STREQ(std::string(p1).c_str(), "foo"); + ASSERT_EQ(std::string(p1), "foo"); } TEST(baseNameOf, pathWithTrailingSlashRoot) { auto p1 = baseNameOf("/"); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return // "" but it actually returns "dir" TEST(baseNameOf, DISABLED_trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_STREQ(std::string(p1).c_str(), ""); + ASSERT_EQ(std::string(p1), ""); } /* ---------------------------------------------------------------------------- @@ -265,11 +265,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(base64Encode, emptyString) { - ASSERT_STREQ(base64Encode("").c_str(), ""); + ASSERT_EQ(base64Encode(""), ""); } TEST(base64Encode, encodesAString) { - ASSERT_STREQ(base64Encode("quod erat demonstrandum").c_str(), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); } TEST(base64Encode, encodeAndDecode) { @@ -277,7 +277,7 @@ namespace nix { auto encoded = base64Encode(s); auto decoded = base64Decode(encoded); - ASSERT_STREQ(decoded.c_str(), s); + ASSERT_EQ(decoded, s); } /* ---------------------------------------------------------------------------- @@ -285,11 +285,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(base64Decode, emptyString) { - ASSERT_STREQ(base64Decode("").c_str(), ""); + ASSERT_EQ(base64Decode(""), ""); } TEST(base64Decode, decodeAString) { - ASSERT_STREQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=").c_str(), "quod erat demonstrandum"); + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } /* ---------------------------------------------------------------------------- @@ -297,12 +297,12 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(toLower, emptyString) { - ASSERT_STREQ(toLower("").c_str(), ""); + ASSERT_EQ(toLower(""), ""); } TEST(toLower, nonLetters) { auto s = "!@(*$#)(@#=\\234_"; - ASSERT_STREQ(toLower(s).c_str(), s); + ASSERT_EQ(toLower(s), s); } // XXX: std::tolower() doesn't cover this. This probably doesn't matter @@ -310,7 +310,7 @@ namespace nix { // characters are not allowed. TEST(toLower, DISABLED_umlauts) { auto s = "ÄÖÜ"; - ASSERT_STREQ(toLower(s).c_str(), "äöü"); + ASSERT_EQ(toLower(s), "äöü"); } /* ---------------------------------------------------------------------------- @@ -377,27 +377,27 @@ namespace nix { StringMap rewrites; rewrites["this"] = "that"; - ASSERT_STREQ(rewriteStrings("", rewrites).c_str(), ""); + ASSERT_EQ(rewriteStrings("", rewrites), ""); } TEST(rewriteStrings, emptyRewrites) { StringMap rewrites; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); } TEST(rewriteStrings, successfulRewrite) { StringMap rewrites; rewrites["this"] = "that"; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "that and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); } TEST(rewriteStrings, doesntOccur) { StringMap rewrites; rewrites["foo"] = "bar"; - ASSERT_STREQ(rewriteStrings("this and that", rewrites).c_str(), "this and that"); + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); } /* ---------------------------------------------------------------------------- @@ -405,16 +405,16 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(replaceStrings, emptyString) { - ASSERT_STREQ(replaceStrings("", "this", "that").c_str(), ""); - ASSERT_STREQ(replaceStrings("this and that", "", "").c_str(), "this and that"); + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); } TEST(replaceStrings, successfulReplace) { - ASSERT_STREQ(replaceStrings("this and that", "this", "that").c_str(), "that and that"); + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); } TEST(replaceStrings, doesntOccur) { - ASSERT_STREQ(replaceStrings("this and that", "foo", "bar").c_str(), "this and that"); + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); } /* ---------------------------------------------------------------------------- @@ -422,14 +422,14 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(trim, emptyString) { - ASSERT_STREQ(trim("").c_str(), ""); + ASSERT_EQ(trim(""), ""); } TEST(trim, removesWhitespace) { - ASSERT_STREQ(trim("foo").c_str(), "foo"); - ASSERT_STREQ(trim(" foo ").c_str(), "foo"); - ASSERT_STREQ(trim(" foo bar baz").c_str(), "foo bar baz"); - ASSERT_STREQ(trim(" \t foo bar baz\n").c_str(), "foo bar baz"); + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); } /* ---------------------------------------------------------------------------- @@ -437,15 +437,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(chomp, emptyString) { - ASSERT_STREQ(chomp("").c_str(), ""); + ASSERT_EQ(chomp(""), ""); } TEST(chomp, removesWhitespace) { - ASSERT_STREQ(chomp("foo").c_str(), "foo"); - ASSERT_STREQ(chomp("foo ").c_str(), "foo"); - ASSERT_STREQ(chomp(" foo ").c_str(), " foo"); - ASSERT_STREQ(chomp(" foo bar baz ").c_str(), " foo bar baz"); - ASSERT_STREQ(chomp("\t foo bar baz\n").c_str(), "\t foo bar baz"); + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); } /* ---------------------------------------------------------------------------- @@ -566,29 +566,29 @@ namespace nix { auto s = ""; auto expected = ""; - ASSERT_STREQ(filterANSIEscapes(s).c_str(), expected); + ASSERT_EQ(filterANSIEscapes(s), expected); } TEST(filterANSIEscapes, doesntChangePrintableChars) { auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - ASSERT_STREQ(filterANSIEscapes(s).c_str(), s); + ASSERT_EQ(filterANSIEscapes(s), s); } TEST(filterANSIEscapes, filtersColorCodes) { auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - ASSERT_STREQ(filterANSIEscapes(s, true, 2).c_str(), " A" ); - ASSERT_STREQ(filterANSIEscapes(s, true, 3).c_str(), " A " ); - ASSERT_STREQ(filterANSIEscapes(s, true, 4).c_str(), " A " ); - ASSERT_STREQ(filterANSIEscapes(s, true, 5).c_str(), " A B" ); - ASSERT_STREQ(filterANSIEscapes(s, true, 8).c_str(), " A B C" ); + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); } TEST(filterANSIEscapes, expandsTabs) { auto s = "foo\tbar\tbaz"; - ASSERT_STREQ(filterANSIEscapes(s, true).c_str(), "foo bar baz" ); + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); } } From 1f3602a2c9c3bfeea7cc8ee4d478c77dec349206 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 18:15:13 +0200 Subject: [PATCH 214/350] Remove replaceInSet The function isn't being used anywhere so it seems safe to remove --- src/libutil/util.hh | 11 ----------- tests/unit-tests/test-util.cc | 4 ---- 2 files changed, 15 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1b263abcc..8770add64 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -389,17 +389,6 @@ string replaceStrings(const std::string & s, std::string rewriteStrings(const std::string & s, const StringMap & rewrites); -/* If a set contains 'from', remove it and insert 'to'. */ -template -void replaceInSet(std::set & set, const T & from, const T & to) -{ - auto i = set.find(from); - if (i == set.end()) return; - set.erase(i); - set.insert(to); -} - - /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index 62ca8f11c..e0738614e 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -364,10 +364,6 @@ namespace nix { ASSERT_EQ(statusOk(1), false); } - /* ---------------------------------------------------------------------------- - * replaceInSet : XXX: this function isn't called anywhere! - * --------------------------------------------------------------------------*/ - /* ---------------------------------------------------------------------------- * rewriteStrings From 73d0b5d807edee05910cff4a30549393d5375d89 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Thu, 7 May 2020 19:29:10 +0200 Subject: [PATCH 215/350] Drop unnecessary std::string --- tests/unit-tests/test-util.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit-tests/test-util.cc b/tests/unit-tests/test-util.cc index e0738614e..a0fa7c109 100644 --- a/tests/unit-tests/test-util.cc +++ b/tests/unit-tests/test-util.cc @@ -111,29 +111,29 @@ namespace nix { TEST(baseNameOf, emptyPath) { auto p1 = baseNameOf(""); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } TEST(baseNameOf, pathOnRoot) { auto p1 = baseNameOf("/dir"); - ASSERT_EQ(std::string(p1), "dir"); + ASSERT_EQ(p1, "dir"); } TEST(baseNameOf, relativePath) { auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(std::string(p1), "foo"); + ASSERT_EQ(p1, "foo"); } TEST(baseNameOf, pathWithTrailingSlashRoot) { auto p1 = baseNameOf("/"); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return // "" but it actually returns "dir" TEST(baseNameOf, DISABLED_trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(std::string(p1), ""); + ASSERT_EQ(p1, ""); } /* ---------------------------------------------------------------------------- From 1b801cec407454b904466153dfc56cec6b433f4b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 7 May 2020 16:43:36 -0600 Subject: [PATCH 216/350] pretending to be const --- src/error-demo/error-demo.cc | 16 ++++++++++++++++ src/libutil/error.hh | 12 ++++++------ src/nix-build/nix-build.cc | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 0d3dcf616..89ba5a78d 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -43,6 +43,22 @@ int main() logError(e.info()); } + // current exception + try { + throw DemoError("DemoError handled as a %1%", "std::exception"); + } + catch (...) { + const std::exception_ptr &eptr = std::current_exception(); + try + { + std::rethrow_exception(eptr); + } + catch (std::exception& e) + { + std::cerr << e.what() << std::endl; + } + } + // For completeness sake, info through vomit levels. // But this is maybe a heavy format for those. logger->logEI( diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 2155ad344..19a806cc1 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -82,10 +82,10 @@ class BaseError : public std::exception { protected: string prefix_; // used for location traces etc. - ErrorInfo err; + mutable ErrorInfo err; - std::optional what_; - const string& calcWhat() + mutable std::optional what_; + const string& calcWhat() const { if (what_.has_value()) return *what_; @@ -131,12 +131,12 @@ public: #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; - const char * what() throw () { return calcWhat().c_str(); } + const char * what() const throw () { return calcWhat().c_str(); } #else - const char * what() noexcept { return calcWhat().c_str(); } + const char * what() const noexcept override { return calcWhat().c_str(); } #endif - const string & msg() { return calcWhat(); } + const string & msg() const { return calcWhat(); } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 401c8d340..3c2f4f00c 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -368,7 +368,7 @@ static void _main(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - // TODO: append error msg + // TODO: append error msg; warn()? logError(e.info()); printError("warning: %s; will use bash from your environment", e.what()); shell = "bash"; From 14073fb76be0f46cb9335edf747fcfa1a5275b7b Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 8 May 2020 12:22:39 +0300 Subject: [PATCH 217/350] Don't block while waiting for build users --- src/libstore/build.cc | 54 +++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1b7e6d75e..00101f553 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -507,9 +507,6 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; - bool findFreeUser(); - - string user; uid_t uid; gid_t gid; @@ -525,20 +522,19 @@ public: uid_t getGID() { assert(gid); return gid; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } + bool findFreeUser(); + bool enabled() { return uid != 0; } }; + UserLock::UserLock() { assert(settings.buildUsersGroup != ""); createDirs(settings.nixStateDir + "/userpool"); - - if (findFreeUser()) return; - - printError("waiting for build users"); - - do std::this_thread::sleep_for(std::chrono::seconds(2)); while (! findFreeUser()); + /* Mark that user is not enabled by default */ + uid = 0; } bool UserLock::findFreeUser() { @@ -1398,6 +1394,30 @@ void DerivationGoal::tryToBuild() { trace("trying to build"); + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique(); + + if (!buildUser->enabled()) { + if (!buildUser->findFreeUser()) { + debug("waiting for build users"); + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -1950,22 +1970,6 @@ void DerivationGoal::startBuilder() #endif } - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - buildUser = std::make_unique(); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); From 772e5db828c9924c803b4d126bca72d7e123614b Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 8 May 2020 12:29:00 +0300 Subject: [PATCH 218/350] Mention build users in the 'waiting for' message --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 00101f553..a8c1cd565 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4830,7 +4830,7 @@ void Worker::waitForInput() if (!waitingForAWhile.empty()) { useTimeout = true; if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); + printError("waiting for locks, build slots or build users..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; timeout = std::max(1L, (long) std::chrono::duration_cast( From 7cc7cef9502dc388706561cbdf275e070520e65d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:34:09 +0200 Subject: [PATCH 219/350] Move unit tests to sr/libutil/tests, use mk make rules --- Makefile | 4 ++-- src/libutil/tests/local.mk | 11 +++++++++++ .../test-util.cc => src/libutil/tests/tests.cc | 4 ++-- tests/unit-tests/local.mk | 4 ---- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 src/libutil/tests/local.mk rename tests/unit-tests/test-util.cc => src/libutil/tests/tests.cc (99%) delete mode 100644 tests/unit-tests/local.mk diff --git a/Makefile b/Makefile index 06de35a27..0ba011e2a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ makefiles = \ local.mk \ nix-rust/local.mk \ src/libutil/local.mk \ + src/libutil/tests/local.mk \ src/libstore/local.mk \ src/libfetchers/local.mk \ src/libmain/local.mk \ @@ -16,8 +17,7 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk \ - tests/unit-tests/local.mk + tests/plugins/local.mk -include Makefile.config diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk new file mode 100644 index 000000000..9c2d2a7bb --- /dev/null +++ b/src/libutil/tests/local.mk @@ -0,0 +1,11 @@ +programs += libutil-tests + +libutil-tests_DIR := $(d) + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_CXXFLAGS += -I src/libutil + +libutil-tests_LIBS = libutil + +libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main) diff --git a/tests/unit-tests/test-util.cc b/src/libutil/tests/tests.cc similarity index 99% rename from tests/unit-tests/test-util.cc rename to src/libutil/tests/tests.cc index a0fa7c109..fb593b5e6 100644 --- a/tests/unit-tests/test-util.cc +++ b/src/libutil/tests/tests.cc @@ -1,5 +1,5 @@ -#include "../../src/libutil/util.hh" -#include "../../src/libutil/types.hh" +#include "util.hh" +#include "types.hh" #include diff --git a/tests/unit-tests/local.mk b/tests/unit-tests/local.mk deleted file mode 100644 index ed11261c1..000000000 --- a/tests/unit-tests/local.mk +++ /dev/null @@ -1,4 +0,0 @@ -LIBS=-larchive -lcrypto -llzma -lbz2 -lz -lbrotlienc -lbrotlidec -unit-tests: - echo $(nix_LDFLAGS) - $(CXX) -o unit-test $(nix_CXXFLAGS) $(nix_LDFLAGS) $(LIBS) --std=c++17 $$(pkg-config --libs gtest_main) ./src/libutil/*.o tests/unit-tests/test-util.cc From 72b9d971bc4ff5b5dcb9349d03526a5ef3e4300e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:35:57 +0200 Subject: [PATCH 220/350] Fix warning --- src/libutil/tests/tests.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index fb593b5e6..0fb4411e8 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -481,8 +481,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(tokenizeString, empty) { - auto s = ""; - Strings expected = { }; + Strings expected = { }; ASSERT_EQ(tokenizeString(""), expected); } From 7898cdb75a721cc21867ccdbcc523e95f2db0354 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 11:49:40 +0200 Subject: [PATCH 221/350] make check: Run unit tests --- mk/programs.mk | 6 ++++++ mk/tracing.mk | 1 + nix-rust/local.mk | 2 +- src/libutil/tests/local.mk | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mk/programs.mk b/mk/programs.mk index d93df4468..2d0c988d4 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -76,4 +76,10 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) + + # Phony target to run this program (typically as a dependency of 'check'). + .PHONY: $(1)_RUN + $(1)_RUN: $$($(1)_PATH) + $(trace-test) $$($(1)_PATH) + endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 13912d3c7..54c77ab60 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -11,6 +11,7 @@ ifeq ($(V), 0) trace-javac = @echo " JAVAC " $@; trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; + trace-test = @echo " TEST " $@; suppress = @ diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 1e006e500..e4bfde31b 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -41,5 +41,5 @@ ifneq ($(OS), Darwin) check: rust-tests rust-tests: - cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) + $(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) endif diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 9c2d2a7bb..a188d5093 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,3 +1,5 @@ +check: libutil-tests_RUN + programs += libutil-tests libutil-tests_DIR := $(d) From ca657525b88e788b72ce53ecc6d140a573476644 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 12:03:27 +0200 Subject: [PATCH 222/350] Don't install unit tests --- mk/programs.mk | 20 ++++++++++++-------- src/libutil/tests/local.mk | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/mk/programs.mk b/mk/programs.mk index 2d0c988d4..3fa9685c3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -35,24 +35,28 @@ define build-program $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + ifdef $(1)_INSTALL_DIR - install: $(DESTDIR)$$($(1)_INSTALL_PATH) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - ifeq ($(BUILD_SHARED_LIBS), 1) + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) - _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + install: $(DESTDIR)$$($(1)_INSTALL_PATH) - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + ifeq ($(BUILD_SHARED_LIBS), 1) + + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) - else + else - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + endif endif # Propagate CFLAGS and CXXFLAGS to the individual object files. diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index a188d5093..5f4b3edc5 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -4,6 +4,8 @@ programs += libutil-tests libutil-tests_DIR := $(d) +libutil-tests_INSTALL_DIR := + libutil-tests_SOURCES := $(wildcard $(d)/*.cc) libutil-tests_CXXFLAGS += -I src/libutil From 5b8883faac0cff928a4359b4260b5be09a311d4b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 May 2020 12:09:00 +0200 Subject: [PATCH 223/350] configure: Look for gtest --- Makefile.config.in | 19 ++++++++++--------- configure.ac | 4 ++++ src/libutil/tests/local.mk | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index e7a12089a..b632444e8 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,36 +1,38 @@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -LDFLAGS = @LDFLAGS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ -HAVE_SODIUM = @HAVE_SODIUM@ +GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -BOOST_LDFLAGS = @BOOST_LDFLAGS@ +HAVE_SODIUM = @HAVE_SODIUM@ +LDFLAGS = @LDFLAGS@ +LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ +LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ -LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ -LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ -LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ -EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ -lsof = @lsof@ datadir = @datadir@ datarootdir = @datarootdir@ +doc_generate = @doc_generate@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ +lsof = @lsof@ mandir = @mandir@ pkglibdir = $(libdir)/$(PACKAGE_NAME) prefix = @prefix@ @@ -38,6 +40,5 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/configure.ac b/configure.ac index b868343d2..c3007b4b6 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,10 @@ if test "$gc" = yes; then fi +# Look for gtest. +PKG_CHECK_MODULES([GTEST], [gtest_main]) + + # documentation generation switch AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], [disable documentation generation]), diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 5f4b3edc5..a297edb64 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -12,4 +12,4 @@ libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $$(pkg-config --libs gtest_main) +libutil-tests_LDFLAGS := $(GTEST_LIBS) From e3df9c2a6e867d0e038cb26af40501511607e007 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:03:44 +0200 Subject: [PATCH 224/350] Enable `dirOf` test Adjusted the doc comment for `dirOf` to reflect the implementation behavior. --- src/libutil/tests/tests.cc | 6 ++---- src/libutil/util.hh | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 0fb4411e8..1b016430d 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -86,12 +86,10 @@ namespace nix { * dirOf * --------------------------------------------------------------------------*/ - // XXX: according to the doc of `dirOf`, dirOf("/") and dirOf("/foo") - // should both return "" but it actually returns "/" in both cases - TEST(dirOf, DISABLED_returnsEmptyStringForRoot) { + TEST(dirOf, returnsEmptyStringForRoot) { auto p = dirOf("/"); - ASSERT_EQ(p, ""); + ASSERT_EQ(p, "/"); } TEST(dirOf, returnsFirstPathComponent) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8770add64..38a0f7a5c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -58,8 +58,8 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means an empty string - is returned. */ + immediate child thereof (e.g., `/foo'), this means `/' + is returned.*/ Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything From 2191141274cfefcb9ffbafbbfcdc58414fb28f1a Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:07:40 +0200 Subject: [PATCH 225/350] Enable `baseNameOf` test Add note about removal of trailing slashes in the doc comment of baseNameOf and enabled the test. --- src/libutil/tests/tests.cc | 6 ++---- src/libutil/util.hh | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 1b016430d..d46f6a5a4 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -127,11 +127,9 @@ namespace nix { ASSERT_EQ(p1, ""); } - // XXX: according to the doc of `baseNameOf`, baseNameOf("/dir/") should return - // "" but it actually returns "dir" - TEST(baseNameOf, DISABLED_trailingSlash) { + TEST(baseNameOf, trailingSlash) { auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, ""); + ASSERT_EQ(p1, "dir"); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 38a0f7a5c..a63ee05b3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -63,7 +63,7 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything - following the final `/'. */ + following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); /* Check whether 'path' is a descendant of 'dir'. */ From 181a47d8845e06edc9653e99de589376a1a96bb6 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Fri, 8 May 2020 15:13:55 +0200 Subject: [PATCH 226/350] Enable toLower umlauts test Update comment and enable the test --- src/libutil/tests/tests.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index d46f6a5a4..5abfe2c9c 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -301,12 +301,12 @@ namespace nix { ASSERT_EQ(toLower(s), s); } - // XXX: std::tolower() doesn't cover this. This probably doesn't matter - // since the context is paths and store paths specifically where such - // characters are not allowed. - TEST(toLower, DISABLED_umlauts) { + // std::tolower() doesn't handle unicode characters. In the context of + // store paths this isn't relevant but doesn't hurt to record this behavior + // here. + TEST(toLower, umlauts) { auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "äöü"); + ASSERT_EQ(toLower(s), "ÄÖÜ"); } /* ---------------------------------------------------------------------------- From 55eb71714854b262b5e1079ff250a13cc0bbf644 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 8 May 2020 18:18:28 -0600 Subject: [PATCH 227/350] add pos to errorinfo, remove from hints --- src/error-demo/error-demo.cc | 17 +- src/libexpr/attr-set.hh | 7 +- src/libexpr/eval-inline.hh | 18 +- src/libexpr/eval.cc | 76 ++++-- src/libexpr/parser.y | 17 +- src/libexpr/primops.cc | 331 +++++++++++++++++++++----- src/libexpr/primops/context.cc | 20 +- src/libexpr/primops/fetchGit.cc | 12 +- src/libexpr/primops/fetchMercurial.cc | 13 +- src/libutil/error.cc | 172 ++++++------- src/libutil/error.hh | 11 +- src/libutil/logging.cc | 3 +- tests/misc.sh | 4 +- 13 files changed, 507 insertions(+), 194 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 89ba5a78d..41293427c 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -126,17 +126,28 @@ int main() // Error with previous and next lines of code. logError( ErrorInfo { .name = "error name", - .description = "error description", + .description = "error with code lines", .hint = hintfmt("this hint has %1% templated %2%!!", "yellow", "values"), .nixCode = NixCode { .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::optional("previous line of code"), + .prevLineOfCode = "previous line of code", .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::optional("next line of code"), + .nextLineOfCode = "next line of code", }}); + // Error without lines of code. + logError( + ErrorInfo { .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + return 0; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 118c7bd5d..f5651891f 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -76,7 +76,12 @@ public: { auto a = get(name); if (!a) - throw Error("attribute '%s' missing, at %s", name, pos); + throw Error( + ErrorInfo { + .hint = hintfmt("attribute '%s' missing", name), + .nixCode = NixCode { .errPos = pos } + }); + return *a; } diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 4d82ccf09..d03633cc7 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -9,7 +9,11 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) { - throw EvalError(s, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) @@ -20,7 +24,11 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) { - throw TypeError(s, showType(v), pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt(s, showType(v)), + .nixCode = NixCode { .errPos = pos } + }); } @@ -43,7 +51,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) else if (v.type == tApp) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered, at %1%", pos); + throwEvalError("infinite recursion encountered", pos); } @@ -59,7 +67,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v); if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected, at %2%", v, pos); + throwTypeError("value is %1% while a set was expected", v, pos); } @@ -75,7 +83,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos) { forceValue(v); if (!v.isList()) - throwTypeError("value is %1% while a list was expected, at %2%", v, pos); + throwTypeError("value is %1% while a list was expected", v, pos); } /* Note: Various places expect the allocated memory to be zeroed. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 516c25b02..a19b85be4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -498,7 +498,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) { - throw EvalError(s, s2, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt(s, s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) @@ -508,7 +512,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) { - throw EvalError(s, s2, s3, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt(s, s2, s3), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) @@ -518,7 +526,11 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, co LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) { - throw TypeError(s, pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) @@ -528,17 +540,29 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) { - throw TypeError(s, fun.showNamePos(), s2, pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt(s, fun.showNamePos(), s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) { - throw AssertionError(s, s1, pos); + throw AssertionError( + ErrorInfo { + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) { - throw UndefinedVarError(s, s1, pos); + throw UndefinedVarError( + ErrorInfo { + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) @@ -804,7 +828,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) Value v; e->eval(*this, env, v); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError("value is %1% while a Boolean was expected", v, pos); return v.boolean; } @@ -1006,7 +1030,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute '%1%' missing, at %2%", name, pos); + throwEvalError("attribute '%1%' missing", name, pos); } vAttrs = j->value; pos2 = j->pos; @@ -1132,7 +1156,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); + throwTypeError("attempt to call something which is not a function but %1%", fun, pos); ExprLambda & lambda(*fun.lambda.fun); @@ -1160,7 +1184,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po for (auto & i : lambda.formals->formals) { Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { - if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%", + if (!i.def) throwTypeError("%1% called without required argument '%2%'", lambda, i.name, pos); env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { @@ -1176,7 +1200,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po user. */ for (auto & i : *arg.attrs) if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos); + throwTypeError("%1% called with unexpected argument '%2%'", lambda, i.name, pos); abort(); // can't happen } } @@ -1417,14 +1441,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); + throwEvalError("cannot add %1% to an integer", showType(vTmp), pos); } else if (firstType == tFloat) { if (vTmp.type == tInt) { nf += vTmp.integer; } else if (vTmp.type == tFloat) { nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); + throwEvalError("cannot add %1% to a float", showType(vTmp), pos); } else s << state.coerceToString(pos, vTmp, context, false, firstType == tString); } @@ -1435,7 +1459,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) mkFloat(v, nf); else if (firstType == tPath) { if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); + throwEvalError("a string that refers to a store path cannot be appended to a path", pos); auto path = canonPath(s.str()); mkPath(v, path.c_str()); } else @@ -1484,7 +1508,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); + throwTypeError("value is %1% while an integer was expected", v, pos); return v.integer; } @@ -1495,7 +1519,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) if (v.type == tInt) return v.integer; else if (v.type != tFloat) - throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + throwTypeError("value is %1% while a float was expected", v, pos); return v.fpoint; } @@ -1504,7 +1528,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError("value is %1% while a Boolean was expected", v, pos); return v.boolean; } @@ -1519,7 +1543,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) - throwTypeError("value is %1% while a function was expected, at %2%", v, pos); + throwTypeError("value is %1% while a function was expected", v, pos); } @@ -1528,7 +1552,7 @@ string EvalState::forceString(Value & v, const Pos & pos) forceValue(v, pos); if (v.type != tString) { if (pos) - throwTypeError("value is %1% while a string was expected, at %2%", v, pos); + throwTypeError("value is %1% while a string was expected", v, pos); else throwTypeError("value is %1% while a string was expected", v); } @@ -1557,7 +1581,7 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) string s = forceString(v, pos); if (v.string.context) { if (pos) - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%", + throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0], pos); else throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", @@ -1614,7 +1638,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return *maybeString; } auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); + if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string", pos); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } @@ -1645,7 +1669,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } } - throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); + throwTypeError("cannot coerce %1% to a string", v, pos); } @@ -1676,7 +1700,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { string path = coerceToString(pos, v, context, false, false); if (path == "" || path[0] != '/') - throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos); + throwEvalError("string '%1%' doesn't represent an absolute path", path, pos); return path; } @@ -1883,7 +1907,11 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { - throw TypeError("cannot coerce %1% to a string, at %2%", showType(), pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt("cannot coerce %1% to a string", showType()), + .nixCode = NixCode { .errPos = pos } + }); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3ed9a7a4f..a3ad54a3d 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -404,7 +404,12 @@ expr_simple | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) - throw ParseError("URL literals are disabled, at %s", CUR_POS); + throw ParseError( + ErrorInfo { + .hint = hintfmt("URL literals are disabled"), + .nixCode = NixCode { .errPos = CUR_POS } + }); + $$ = new ExprString(data->symbols.create($1)); } | '(' expr ')' { $$ = $2; } @@ -669,10 +674,12 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - string f = - "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)" - + string(pos ? ", at %2%" : ""); - throw ThrownError(f, path, pos); + + throw ThrownError( + ErrorInfo { + .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), + .nixCode = NixCode { .errPos = pos } + }); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 13eb1ba58..771136af9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -94,8 +94,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%", - path, e.path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -171,8 +176,13 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError("cannot import '%1%', since path '%2%' is not valid, at %3%" - , path, e.path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt( + "cannot import '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } path = state.checkSourcePath(path); @@ -207,7 +217,11 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) { - throw EvalError("at least one argument to 'exec' required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("at least one argument to 'exec' required"), + .nixCode = NixCode { .errPos = pos } + }); } PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); @@ -218,9 +232,12 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError("cannot execute '%1%', since path '%2%' is not valid, at %3%" - , program, e.path, pos); - } + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid" + , program, e.path), + .nixCode = NixCode { .errPos = pos } + });} auto output = runProgram(program, true, commandArgs); Expr * parsed; @@ -371,7 +388,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator startSet = args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) - throw EvalError("attribute 'startSet' required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("attribute 'startSet' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceList(*startSet->value, pos); ValueList workSet; @@ -382,7 +403,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator op = args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) - throw EvalError("attribute 'operator' required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("attribute 'operator' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*op->value); /* Construct the closure by applying the operator to element of @@ -401,7 +426,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator key = e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) - throw EvalError("attribute 'key' required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("attribute 'key' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*key->value); if (!doneKeys.insert(key->value).second) continue; @@ -533,7 +562,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Figure out the name first (for stack backtraces). */ Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) - throw EvalError("required attribute 'name' missing, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("required attribute 'name' missing"), + .nixCode = NixCode { .errPos = pos } + }); string drvName; Pos & posDrvName(*attr->pos); try { @@ -576,25 +609,45 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto handleHashMode = [&](const std::string & s) { if (s == "recursive") outputHashRecursive = true; else if (s == "flat") outputHashRecursive = false; - else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); + else + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .nixCode = NixCode { .errPos = posDrvName } + }); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - throw EvalError("duplicate derivation output '%1%', at %2%", j, posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("duplicate derivation output '%1%'", j), + .nixCode = NixCode { .errPos = posDrvName } + }); + + /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - throw EvalError("invalid derivation output name 'drv', at %1%", posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid derivation output name 'drv'" ), + .nixCode = NixCode { .errPos = posDrvName } + }); + outputs.insert(j); } if (outputs.empty()) - throw EvalError("derivation cannot have an empty set of outputs, at %1%", posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("derivation cannot have an empty set of outputs"), + .nixCode = NixCode { .errPos = posDrvName } + }); }; try { @@ -706,18 +759,37 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError("required attribute 'builder' missing, at %1%", posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("required attribute 'builder' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); + if (drv.platform == "") - throw EvalError("required attribute 'system' missing, at %1%", posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("required attribute 'system' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); + /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) - throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName); + throw EvalError( + ErrorInfo { + .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .nixCode = NixCode { .errPos = posDrvName } + }); + if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error("multiple outputs are not supported in fixed-output derivations, at %1%", posDrvName); + throw Error( + ErrorInfo { + .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .nixCode = NixCode { .errPos = posDrvName } + }); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); @@ -818,7 +890,11 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) - throw EvalError("path '%1%' is not in the Nix store, at %2%", path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("path '%1%' is not in the Nix store", path), + .nixCode = NixCode { .errPos = pos } + }); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(path2)); @@ -835,8 +911,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError( - "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%", - path, e.path, pos); + ErrorInfo { + .hint = hintfmt( + "cannot check the existence of '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } try { @@ -879,9 +959,13 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%" - , path, e.path, pos); - } + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid" + , path, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); @@ -908,7 +992,11 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) - throw EvalError("attribute 'path' missing, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("attribute 'path' missing"), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -916,8 +1004,12 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError("cannot find '%1%', since path '%2%' is not valid, at %3%", - path, e.path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } searchPath.emplace_back(prefix, path); @@ -934,7 +1026,11 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error("unknown hash type '%1%', at %2%", type, pos); + throw Error( + ErrorInfo { + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded Path p = state.coerceToPath(pos, *args[1], context); @@ -950,8 +1046,12 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val try { state.realiseContext(ctx); } catch (InvalidPathError & e) { - throw EvalError("cannot read '%1%', since path '%2%' is not valid, at %3%", - path, e.path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } DirEntries entries = readDirectory(state.checkSourcePath(path)); @@ -1021,7 +1121,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%", name, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt( + "in 'toFile': the file '%1%' cannot refer to derivation outputs", \ + name), + .nixCode = NixCode { .errPos = pos } + }); refs.insert(state.store->parseStorePath(path)); } @@ -1089,11 +1195,21 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) - throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*args[0]); if (args[0]->type != tLambda) - throw TypeError("first argument in call to 'filterSource' is not a function but %1%, at %2%", showType(*args[0]), pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt( + "first argument in call to 'filterSource' is not a function but %1%", + showType(*args[0])), + .nixCode = NixCode { .errPos = pos } + }); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); } @@ -1113,7 +1229,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value PathSet context; path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) - throw EvalError("string '%1%' cannot refer to other paths, at %2%", path, *attr.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("string '%1%' cannot refer to other paths", + path), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { @@ -1124,10 +1246,19 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else - throw EvalError("unsupported argument '%1%' to 'addPath', at %2%", attr.name, *attr.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("unsupported argument '%1%' to 'addPath'", + attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (path.empty()) - throw EvalError("'path' required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("'path' required"), + .nixCode = NixCode { .errPos = pos } + }); if (name.empty()) name = baseNameOf(path); @@ -1185,7 +1316,12 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - throw EvalError("attribute '%1%' missing, at %2%", attr, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("attribute '%1%' missing", attr), + .nixCode = NixCode { .errPos = pos } + }); + // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value); @@ -1265,14 +1401,23 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) - throw TypeError("'name' attribute missing in a call to 'listToAttrs', at %1%", pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); + string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) - throw TypeError("'value' attribute missing in a call to 'listToAttrs', at %1%", pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } @@ -1346,7 +1491,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args { state.forceValue(*args[0]); if (args[0]->type != tLambda) - throw TypeError("'functionArgs' requires a function, at %1%", pos); + throw TypeError( + ErrorInfo { + .hint = hintfmt("'functionArgs' requires a function"), + .nixCode = NixCode { .errPos = pos } + }); + if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); @@ -1396,7 +1546,12 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error("list index %1% is out of bounds, at %2%", n, pos); + throw Error( + ErrorInfo { + .hint = hintfmt("list index %1% is out of bounds", n), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*list.listElems()[n]); v = *list.listElems()[n]; } @@ -1423,7 +1578,12 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value { state.forceList(*args[0], pos); if (args[0]->listSize() == 0) - throw Error("'tail' called on an empty list, at %1%", pos); + throw Error( + ErrorInfo { + .hint = hintfmt("'tail' called on an empty list"), + .nixCode = NixCode { .errPos = pos } + }); + state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) v.listElems()[n] = args[0]->listElems()[n + 1]; @@ -1564,7 +1724,12 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val auto len = state.forceInt(*args[1], pos); if (len < 0) - throw EvalError("cannot create list of size %1%, at %2%", len, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("cannot create list of size %1%", len), + .nixCode = NixCode { .errPos = pos } + }); + state.mkList(v, len); @@ -1722,7 +1887,12 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[1], pos); NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) throw EvalError("division by zero, at %1%", pos); + if (f2 == 0) + throw EvalError( + ErrorInfo { + .hint = hintfmt("division by zero"), + .nixCode = NixCode { .errPos = pos } + }); if (args[0]->type == tFloat || args[1]->type == tFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); @@ -1731,7 +1901,12 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixInt i2 = state.forceInt(*args[1], pos); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - throw EvalError("overflow in integer division, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("overflow in integer division"), + .nixCode = NixCode { .errPos = pos } + }); + mkInt(v, i1 / i2); } } @@ -1787,7 +1962,12 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V PathSet context; string s = state.coerceToString(pos, *args[2], context); - if (start < 0) throw EvalError("negative start position in 'substring', at %1%", pos); + if (start < 0) + throw EvalError( + ErrorInfo { + .hint = hintfmt("negative start position in 'substring'"), + .nixCode = NixCode { .errPos = pos } + }); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } @@ -1807,7 +1987,11 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error("unknown hash type '%1%', at %2%", type, pos); + throw Error( + ErrorInfo { + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); @@ -1849,10 +2033,18 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError( + ErrorInfo { + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -1917,10 +2109,18 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); - } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); - } + throw EvalError( + ErrorInfo { + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } else { + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } } } @@ -1950,7 +2150,11 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[0], pos); state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .nixCode = NixCode { .errPos = pos } + }); vector from; from.reserve(args[0]->listSize()); @@ -2072,11 +2276,20 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, else if (n == "name") request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%1%' to '%2%', at %3%", attr.name, who, attr.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("unsupported argument '%1%' to '%2%'", attr.name, who), + .nixCode = NixCode { .errPos = pos } + }); + } if (request.uri.empty()) - throw EvalError("'url' argument required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else request.uri = state.forceStringNoCtx(*args[0], pos); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 94fa0158c..768936453 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -146,7 +146,12 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg auto sAllOutputs = state.symbols.create("allOutputs"); for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) - throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("Context key '%s' is not a store path", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); + if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(i.name)); state.forceAttrs(*i.value, *i.pos); @@ -160,7 +165,12 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { - throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); + } context.insert("=" + string(i.name)); } @@ -170,7 +180,11 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { - throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); } for (unsigned int n = 0; n < iter->value->listSize(); ++n) { auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 480cc1adc..68ffed513 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -217,11 +217,19 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (url.empty()) - throw EvalError("'url' argument required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 76029f548..ce99928e2 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -187,11 +187,20 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } if (url.empty()) - throw EvalError("'url' argument required, at %1%", pos); + throw EvalError( + ErrorInfo { + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index d4305ddd8..ce5dde6c3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,4 @@ -#include "types.hh" +#include "error.hh" #include #include @@ -25,54 +25,60 @@ std::ostream& operator<<(std::ostream &os, const hintformat &hf) string showErrPos(const ErrPos &errPos) { - if (errPos.column > 0) { - return fmt("(%1%:%2%)", errPos.line, errPos.column); - } else { - return fmt("(%1%)", errPos.line); - }; + if (errPos.line > 0) { + if (errPos.column > 0) { + return fmt("(%1%:%2%)", errPos.line, errPos.column); + } else { + return fmt("(%1%)", errPos.line); + } + } + else { + return ""; + } } -void printCodeLines(const string &prefix, const NixCode &nixCode) +void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode) { // previous line of code. if (nixCode.prevLineOfCode.has_value()) { - std::cout << fmt("%1% %|2$5d|| %3%", + out << fmt("%1% %|2$5d|| %3%", prefix, (nixCode.errPos.line - 1), *nixCode.prevLineOfCode) - << std::endl; + << std::endl; } - // line of code containing the error.%2$+5d% - std::cout << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line), - nixCode.errLineOfCode) - << std::endl; - - // error arrows for the column range. - if (nixCode.errPos.column > 0) { - int start = nixCode.errPos.column; - std::string spaces; - for (int i = 0; i < start; ++i) { - spaces.append(" "); - } - - std::string arrows("^"); - - std::cout << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + if (nixCode.errLineOfCode.has_value()) { + // line of code containing the error. + out << fmt("%1% %|2$5d|| %3%", prefix, - spaces, - arrows) << std::endl; + (nixCode.errPos.line), + *nixCode.errLineOfCode) + << std::endl; + // error arrows for the column range. + if (nixCode.errPos.column > 0) { + int start = nixCode.errPos.column; + std::string spaces; + for (int i = 0; i < start; ++i) { + spaces.append(" "); + } + + std::string arrows("^"); + + out << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows) << std::endl; + } } // next line of code. if (nixCode.nextLineOfCode.has_value()) { - std::cout << fmt("%1% %|2$5d|| %3%", + out << fmt("%1% %|2$5d|| %3%", prefix, (nixCode.errPos.line + 1), *nixCode.nextLineOfCode) - << std::endl; + << std::endl; } } @@ -83,52 +89,52 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) string levelString; switch (einfo.level) { - case Verbosity::lvlError: { - levelString = ANSI_RED; - levelString += "error:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlWarn: { - levelString = ANSI_YELLOW; - levelString += "warning:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlInfo: { - levelString = ANSI_GREEN; - levelString += "info:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlTalkative: { - levelString = ANSI_GREEN; - levelString += "talk:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlChatty: { - levelString = ANSI_GREEN; - levelString += "chat:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlVomit: { - levelString = ANSI_GREEN; - levelString += "vomit:"; - levelString += ANSI_NORMAL; - break; - } - case Verbosity::lvlDebug: { - levelString = ANSI_YELLOW; - levelString += "debug:"; - levelString += ANSI_NORMAL; - break; - } - default: { - levelString = fmt("invalid error level: %1%", einfo.level); - break; - } + case Verbosity::lvlError: { + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlWarn: { + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlInfo: { + levelString = ANSI_GREEN; + levelString += "info:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlTalkative: { + levelString = ANSI_GREEN; + levelString += "talk:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlChatty: { + levelString = ANSI_GREEN; + levelString += "chat:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlVomit: { + levelString = ANSI_GREEN; + levelString += "vomit:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlDebug: { + levelString = ANSI_YELLOW; + levelString += "debug:"; + levelString += ANSI_NORMAL; + break; + } + default: { + levelString = fmt("invalid error level: %1%", einfo.level); + break; + } } int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); @@ -158,14 +164,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) // filename. if (einfo.nixCode.has_value()) { if (einfo.nixCode->errPos.file != "") { - string eline = einfo.nixCode->errLineOfCode != "" - ? string(" ") + showErrPos(einfo.nixCode->errPos) - : ""; - - out << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL, + out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, prefix, einfo.nixCode->errPos.file, - eline) << std::endl; + showErrPos(einfo.nixCode->errPos)) << std::endl; out << prefix << std::endl; } else { out << fmt("%1%from command line argument", prefix) << std::endl; @@ -180,8 +182,8 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) } // lines of code. - if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode != "") { - printCodeLines(prefix, *einfo.nixCode); + if (einfo.nixCode.has_value()) { + printCodeLines(out, prefix, *einfo.nixCode); out << prefix << std::endl; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 19a806cc1..7e2cca2f9 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -35,10 +35,15 @@ typedef enum { } Verbosity; struct ErrPos { - int line; - int column; + int line = 0; + int column = 0; string file; + operator bool() const + { + return line != 0; + } + template ErrPos& operator=(const P &pos) { @@ -58,7 +63,7 @@ struct ErrPos { struct NixCode { ErrPos errPos; std::optional prevLineOfCode; - string errLineOfCode; + std::optional errLineOfCode; std::optional nextLineOfCode; }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index e1750eb2a..4244d1221 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -171,7 +171,8 @@ struct JSONLogger : Logger { json["file"] = ei.nixCode->errPos.file; if (ei.nixCode->prevLineOfCode.has_value()) json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode; - json["errLineOfCode"] = ei.nixCode->errLineOfCode; + if (ei.nixCode->errLineOfCode.has_value()) + json["errLineOfCode"] = *ei.nixCode->errLineOfCode; if (ei.nixCode->nextLineOfCode.has_value()) json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode; } diff --git a/tests/misc.sh b/tests/misc.sh index eda016416..fd4908e25 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,6 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" +eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) +echo $eval_res | grep "(string) (1:15)" +echo $eval_res | grep "infinite recursion encountered" From 52cffafd2462c03466db15909dde250f07677d35 Mon Sep 17 00:00:00 2001 From: Dani Date: Sat, 9 May 2020 13:48:31 +0200 Subject: [PATCH 228/350] Fix typo --- doc/manual/release-notes/rl-0.8.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml index 784b26c6b..825798fa9 100644 --- a/doc/manual/release-notes/rl-0.8.xml +++ b/doc/manual/release-notes/rl-0.8.xml @@ -8,7 +8,7 @@ NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a result, nix-pull manifests and channels built -for Nix 0.7 and below will now work anymore. However, the Nix +for Nix 0.7 and below will not work anymore. However, the Nix expression language has not changed, so you can still build from source. Also, existing user environments continue to work. Nix 0.8 will automatically upgrade the database schema of previous From 446649e5403f9c41601eee6438eaeed5212b9190 Mon Sep 17 00:00:00 2001 From: Shao Cheng Date: Sat, 9 May 2020 15:59:39 +0200 Subject: [PATCH 229/350] Update "Upgrading Nix" documentation This PR proposes two changes to the "Upgrading Nix" documentation: * Besides updating `nixpkgs.nix`, we also update `nixpkgs.cacert`, so that the certificates are up-to-date as well. * Add the instructions for multi-user mode on Linux. --- doc/manual/installation/upgrading.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml index 30670d7fe..592f63895 100644 --- a/doc/manual/installation/upgrading.xml +++ b/doc/manual/installation/upgrading.xml @@ -17,6 +17,11 @@ Single-user installations of Nix should run this: - nix-channel --update; nix-env -iA nixpkgs.nix + nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert + + + + Multi-user Nix users on Linux should run this with sudo: + nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon From 146f9c114ff347d372b5ab6c5fc00fdd3f2895da Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 9 May 2020 10:58:43 -0400 Subject: [PATCH 230/350] doc: consistently refer to 'fixed-output' with a dash General cleanup that makes it easier to search for the term. --- doc/manual/command-ref/nix-store.xml | 2 +- src/libstore/build.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 1ddb5408d..d71f9d8e4 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -936,7 +936,7 @@ $ nix-store --add ./foo.c The operation adds the specified paths to the Nix store. Unlike paths are registered using the -specified hashing algorithm, resulting in the same output path as a fixed output +specified hashing algorithm, resulting in the same output path as a fixed-output derivation. This can be used for sources that are not available from a public url or broke since the download expression was written. diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 147093fae..6258bcb33 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3155,7 +3155,7 @@ void DerivationGoal::runChild() // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the - // potential impurities introduced in fixed outputs. + // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); ss.push_back("/etc/services"); From 1d8144e36b18d3467f1b29b80800ed3eb9b876c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 11 May 2020 18:14:23 +0200 Subject: [PATCH 231/350] Update src/libstore/build.cc --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e0c7324c7..f9b701910 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3679,7 +3679,7 @@ void DerivationGoal::registerOutputs() if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( format("output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat).") % path); + "since recursive hashing is not enabled (outputHashMode=flat)") % path); } /* Check the hash. In hash mode, move the path produced by From 958e81987b5b86fba06090078a556f51037aa23c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 May 2020 13:02:16 -0600 Subject: [PATCH 232/350] switch from printError warnings to logWarnings --- src/libstore/build.cc | 14 +++++++++++--- src/libstore/store-api.cc | 6 +++++- src/libutil/error.cc | 2 +- src/libutil/serialise.cc | 5 ++++- src/nix-build/nix-build.cc | 9 ++++++--- src/nix-store/nix-store.cc | 5 ++++- 6 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f8cc1ce36..a7d6e53b0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3647,7 +3647,11 @@ void DerivationGoal::registerOutputs() /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError("warning: rewriting hashes in '%1%'; cross fingers", path); + logWarning( + ErrorInfo { + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -4414,8 +4418,12 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)); + logWarning( + ErrorInfo { + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 542e5c552..d1281d130 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -769,7 +769,11 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + logWarning( + ErrorInfo{ + .name = "Path not content-addressed", + .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) + }); }; if (hasPrefix(ca, "text:")) { diff --git a/src/libutil/error.cc b/src/libutil/error.cc index ce5dde6c3..e4f45a2d3 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -161,7 +161,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) einfo.programName.value_or("")) << std::endl; - // filename. + // filename, line, column. if (einfo.nixCode.has_value()) { if (einfo.nixCode->errPos.file != "") { out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd..35f7ee917 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -52,7 +52,10 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); + logWarning(ErrorInfo { + .name = "Large path", + .description = "dumping very large path (> 256 MiB); this may run out of memory" + }); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3c2f4f00c..03200d050 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -368,9 +368,12 @@ static void _main(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - // TODO: append error msg; warn()? - logError(e.info()); - printError("warning: %s; will use bash from your environment", e.what()); + logWarning( + ErrorInfo { + .name = "bashInteractive", + .hint = hintfmt("%s; will use bash from your environment", + (e.info().hint ? e.info().hint->str() : "")) + }); shell = "bash"; } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 57063d42f..9b5cceccf 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -704,7 +704,10 @@ static void opVerify(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - printError("warning: not all errors were fixed"); + logWarning(ErrorInfo { + .name = "Store consistency", + .description = "not all errors were fixed" + }); throw Exit(1); } } From 536bbf53e12c80f52c2679aec734d895b0058f5b Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 May 2020 13:58:38 -0600 Subject: [PATCH 233/350] comments and cleanup --- src/error-demo/error-demo.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 41293427c..8f484431c 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -10,27 +10,27 @@ MakeError(DemoError, Error); int main() { - makeDefaultLogger(); - verbosity = lvlVomit; // In each program where errors occur, this has to be set. ErrorInfo::programName = std::optional("error-demo"); + // 'DemoError' appears as error name. try { throw DemoError("demo error was thrown"); } catch (Error &e) { logger->logEI(e.info()); } - // ErrorInfo constructor + // appending to the hint from the previous error try { - auto e = Error("some error"); + auto e = Error("initial error"); throw DemoError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - string prevhint = (e.info().hint.has_value() ? e.info().hint->str() : ""); - ei.hint = std::optional(hintfmt("previous hint was: %s", normaltxt(prevhint))); + // using normaltxt to avoid the default yellow highlighting. + ei.hint = hintfmt("%s; subsequent error message.", + normaltxt(e.info().hint ? e.info().hint->str() : "")); logger->logEI(ei); } @@ -38,8 +38,8 @@ int main() try { auto x = readFile(-1); } - catch (Error &e) { - std::cout << "error: " << e.sname() << std::endl; + catch (SysError &e) { + std::cout << "errno was: " << e.errNo << std::endl; logError(e.info()); } @@ -59,7 +59,7 @@ int main() } } - // For completeness sake, info through vomit levels. + // For completeness sake, show 'info' through 'vomit' levels. // But this is maybe a heavy format for those. logger->logEI( ErrorInfo { .level = lvlInfo, From b93c1bf3d67716231d2ff7ee4154b8c80a251b10 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 May 2020 15:52:15 -0600 Subject: [PATCH 234/350] fixes to merged code --- src/libexpr/nixexpr.hh | 1 + src/libexpr/primops.cc | 2 +- src/libmain/stack.cc | 2 +- src/libstore/build.cc | 20 ++++++++++---------- src/libstore/filetransfer.hh | 3 ++- src/libstore/local-store.cc | 1 + src/libstore/sqlite.hh | 2 +- src/libutil/args.cc | 7 ------- src/libutil/error.hh | 4 +--- src/libutil/logging.hh | 1 + src/libutil/types.hh | 1 + src/libutil/url.hh | 2 +- src/libutil/util.cc | 16 ++++++++-------- 13 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 137fe7198..a185d5785 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -2,6 +2,7 @@ #include "value.hh" #include "symbol-table.hh" +#include "error.hh" #include diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 10e8c6b42..657ce3001 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1124,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu ErrorInfo { .hint = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " - "to a derivation but contains (%2%)" + "to a derivation but contains (%2%)", name, path), .nixCode = NixCode { .errPos = pos } diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index e6224de7d..b0a4a4c5d 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -1,4 +1,4 @@ -#include "types.hh" +#include "error.hh" #include #include diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d6f3553d9..0359b10ac 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -547,7 +547,7 @@ UserLock::UserLock() /* Copy the result of getgrnam. */ Strings users; for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user '%1%'") % *p); + debug("found build user '%1%'", *p); users.push_back(*p); } @@ -558,7 +558,7 @@ UserLock::UserLock() /* Find a user account that isn't currently in use for another build. */ for (auto & i : users) { - debug(format("trying user '%1%'") % i); + debug("trying user '%1%'", i); struct passwd * pw = getpwnam(i.c_str()); if (!pw) @@ -1794,7 +1794,7 @@ HookReply DerivationGoal::tryBuildHook() } } - debug(format("hook reply is '%1%'") % reply); + debug("hook reply is '%1%'", reply); if (reply == "decline") return rpDecline; @@ -2255,7 +2255,7 @@ void DerivationGoal::startBuilder() startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -3195,7 +3195,7 @@ void DerivationGoal::runChild() filesystem that we want in the chroot environment. */ auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug(format("bind mounting '%1%' to '%2%'") % source % target); + debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; if (stat(source.c_str(), &st) == -1) { if (optional && errno == ENOENT) @@ -3572,7 +3572,7 @@ static void moveCheckToStore(const Path & src, const Path & dst) directory's parent link ".."). */ struct stat st; if (lstat(src.c_str(), &st) == -1) { - throw SysError(format("getting attributes of path '%1%'") % src); + throw SysError("getting attributes of path '%1%'", src); } bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); @@ -3581,7 +3581,7 @@ static void moveCheckToStore(const Path & src, const Path & dst) chmod_(src, st.st_mode | S_IWUSR); if (rename(src.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % src % dst); + throw SysError("renaming '%1%' to '%2%'", src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -4911,15 +4911,15 @@ void Worker::waitForInput() // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { - debug(format("%1%: got EOF") % goal->getName()); + debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), rd); string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 517b1a7d3..11dca2fe0 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,8 +103,9 @@ class FileTransferError : public Error { public: FileTransfer::Error error; + template FileTransferError(FileTransfer::Error error, const Args & ... args) - : Error(fs), error(error) + : Error(args...), error(error) { } }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1f16a22b1..eb7548543 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" +#include "error.hh" #include #include diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index ce27033c9..ccfac48d8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -3,7 +3,7 @@ #include #include -#include "types.hh" +#include "error.hh" struct sqlite3; struct sqlite3_stmt; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 423b99c96..10d6e89bb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -101,15 +101,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) std::vector args; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { -<<<<<<< HEAD - if (flag.arity == ArityAny) break; - throw UsageError("flag '%1%' requires %2% argument(s)", - name, - flag.arity); -======= if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); ->>>>>>> master } args.push_back(*pos++); } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7e2cca2f9..19c6e35a1 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -2,6 +2,7 @@ #include "ref.hh" +#include "types.hh" #include #include @@ -21,9 +22,6 @@ namespace nix { -using std::list; -using std::vector; - typedef enum { lvlError = 0, lvlWarn, diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index bb7d356c8..39692a291 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "error.hh" namespace nix { diff --git a/src/libutil/types.hh b/src/libutil/types.hh index a831b9924..4d783b564 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -13,6 +13,7 @@ namespace nix { using std::list; using std::set; using std::vector; +using std::string; typedef list Strings; typedef set StringSet; diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 1503023a2..a184eadce 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -1,6 +1,6 @@ #pragma once -#include "types.hh" +#include "error.hh" #include diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8207ff701..ac7c2967b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -276,7 +276,7 @@ DirEntries readDirectory(DIR *dir, const Path & path) DirEntries readDirectory(const Path & path) { AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); return readDirectory(dir.get(), path); } @@ -306,7 +306,7 @@ string readFile(const Path & path) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); return readFile(fd.get()); } @@ -394,15 +394,15 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError(format("chmod '%1%'") % path); + throw SysError("chmod '%1%'", path); } int fd = openat(parentfd, path.c_str(), O_RDONLY); if (!fd) - throw SysError(format("opening directory '%1%'") % path); + throw SysError("opening directory '%1%'", path); AutoCloseDir dir(fdopendir(fd)); if (!dir) - throw SysError(format("opening directory '%1%'") % path); + throw SysError("opening directory '%1%'", path); for (auto & i : readDirectory(dir.get(), path)) _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); } @@ -426,7 +426,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) // for backwards compatibility. if (errno == ENOENT) return; - throw SysError(format("opening directory '%1%'") % path); + throw SysError("opening directory '%1%'", path); } _deletePath(dirfd.get(), path, bytesFreed); @@ -845,7 +845,7 @@ int Pid::kill() { assert(pid != -1); - debug(format("killing process %1%") % pid); + debug("killing process %1%", pid); /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child @@ -903,7 +903,7 @@ pid_t Pid::release() void killUser(uid_t uid) { - debug(format("killing all processes running under uid '%1%'") % uid); + debug("killing all processes running under uid '%1%'", uid); assert(uid != 0); /* just to be safe... */ From 631642c5b40032a087b8e2e131b2ebcac00913d3 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 May 2020 16:58:08 -0600 Subject: [PATCH 235/350] new format for pos --- src/libexpr/parser.y | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 82d5753ab..6d50d74ba 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,15 +64,23 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { - throw ParseError("attribute '%1%' at %2% already defined at %3%", - showAttrPath(attrPath), pos, prevPos); + throw ParseError( + ErrorInfo { + .hint = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { - throw ParseError("attribute '%1%' at %2% already defined at %3%", - attr, pos, prevPos); + throw ParseError( + ErrorInfo { + .hint = hintfmt("attribute '%1%' already defined at %2%", + attr, prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } @@ -140,8 +148,12 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) - throw ParseError("duplicate formal function argument '%1%' at %2%", - formal.name, pos); + throw ParseError( + ErrorInfo { + .hint = hintfmt("duplicate formal function argument '%1%'", + formal.name), + .nixCode = NixCode { .errPos = pos }, + }); formals->formals.push_front(formal); } @@ -327,7 +339,11 @@ expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) - throw ParseError("dynamic attributes not allowed in let at %1%", CUR_POS); + throw ParseError( + ErrorInfo { + .hint = hintfmt("dynamic attributes not allowed in let"), + .nixCode = NixCode { .errPos = CUR_POS }, + }); $$ = new ExprLet($2, $4); } | expr_if @@ -479,8 +495,11 @@ attrs $$->push_back(AttrName(str->s)); delete str; } else - throw ParseError("dynamic attributes not allowed in inherit at %1%", - makeCurPos(@2, data)); + throw ParseError( + ErrorInfo { + .hint = hintfmt("dynamic attributes not allowed in inherit"), + .nixCode = NixCode { .errPos = makeCurPos(@2, data) }, + }); } | { $$ = new AttrPath; } ; From 7c3138844ca9ab92e2f231a7d17d7aee420ae76e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 11 May 2020 17:34:57 -0600 Subject: [PATCH 236/350] more pos reporting --- src/libexpr/nixexpr.cc | 8 ++++++-- src/libexpr/parser.y | 8 +++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index a9955f6df..91a508305 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -267,8 +267,12 @@ void ExprVar::bindVars(const StaticEnv & env) /* Otherwise, the variable must be obtained from the nearest enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError("undefined variable '%1%' at %2%", name, pos); - + if (withLevel == -1) + throw UndefinedVarError( + ErrorInfo { + .hint = hintfmt("undefined variable '%1%'", name), + .nixCode = NixCode { .errPos = pos } + }); fromWith = true; this->level = withLevel; } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6d50d74ba..3ae7bbafd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,7 +31,7 @@ namespace nix { Expr * result; Path basePath; Symbol path; - string error; + ErrorInfo error; Symbol sLetBody; ParseData(EvalState & state) : state(state) @@ -261,8 +261,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { - data->error = (format("%1%, at %2%") - % error % makeCurPos(*loc, data)).str(); + data->error = ErrorInfo { + .hint = hintfmt(error), + .nixCode = NixCode { .errPos = makeCurPos(*loc, data) } + }; } From 46be11b762d6048746728e6e546dd4f0fac581fc Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Tue, 12 May 2020 12:13:40 +0200 Subject: [PATCH 237/350] Introduce NIX_INSTALLER_NO_CHANNEL_ADD which skips nix-channel --add --- scripts/install-nix-from-closure.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index e00708f6c..21915b37d 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -130,13 +130,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then fi # Subscribe the user to the Nixpkgs channel and fetch it. -if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable -fi -if [ -z "$_NIX_INSTALLER_TEST" ]; then - if ! $nix/bin/nix-channel --update nixpkgs; then - echo "Fetching the nixpkgs channel failed. (Are you offline?)" - echo "To try again later, run \"nix-channel --update nixpkgs\"." +if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + fi + if [ -z "$_NIX_INSTALLER_TEST" ]; then + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi fi fi From 5722f9690c39d136ffbf452f3b85df09e8127712 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 May 2020 13:49:55 +0200 Subject: [PATCH 238/350] tests/binary-cache.sh: Improve incomplete closure test Issue #3373. --- tests/binary-cache.sh | 18 ++++++++++++++++-- tests/build-hook.nix | 4 ++-- tests/build-remote.sh | 4 ++-- tests/config.nix.in | 2 +- tests/dependencies.builder1.sh | 2 -- tests/dependencies.builder2.sh | 2 -- tests/dependencies.nix | 15 ++++++++++++--- tests/dependencies.sh | 2 +- tests/export-graph.sh | 2 +- tests/gc-concurrent.nix | 6 +++--- tests/nix-channel.sh | 8 ++++---- 11 files changed, 42 insertions(+), 23 deletions(-) delete mode 100644 tests/dependencies.builder1.sh delete mode 100644 tests/dependencies.builder2.sh diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index a3c3c7847..17b63d978 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -105,10 +105,24 @@ mv $cacheDir/nar2 $cacheDir/nar # incomplete closure. clearStore -rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "copying path" $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*input-2" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log + + +# Idem, but without cached .narinfo. +clearStore +clearCacheCache + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "don't know how to build" $TEST_ROOT/log +grep -q "building.*input-1" $TEST_ROOT/log +grep -q "building.*input-2" $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log if [ -n "$HAVE_SODIUM" ]; then diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 8bff0fe79..8c5ca8cd3 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -4,13 +4,13 @@ let input1 = mkDerivation { name = "build-hook-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { name = "build-hook-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; in diff --git a/tests/build-remote.sh b/tests/build-remote.sh index ddd68f327..a550f4460 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -20,5 +20,5 @@ cat $outPath/foobar | grep FOOBAR # Ensure that input1 was built on store1 due to the required feature. p=$(readlink -f $outPath/input-2) -(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh) -nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh +(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) +nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh diff --git a/tests/config.nix.in b/tests/config.nix.in index 0ec2eba6b..a57a8c596 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -11,7 +11,7 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/dependencies.builder1.sh b/tests/dependencies.builder1.sh deleted file mode 100644 index 4b006a17d..000000000 --- a/tests/dependencies.builder1.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo FOO > $out/foo diff --git a/tests/dependencies.builder2.sh b/tests/dependencies.builder2.sh deleted file mode 100644 index 4f886fdb3..000000000 --- a/tests/dependencies.builder2.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo BAR > $out/bar diff --git a/tests/dependencies.nix b/tests/dependencies.nix index eca4b2964..e320d81c9 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -2,18 +2,27 @@ with import ./config.nix; let { + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = "${./dependencies.builder2.sh}"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + ''; }; body = mkDerivation { - name = "dependencies"; + name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; diff --git a/tests/dependencies.sh b/tests/dependencies.sh index 8d0fdc10f..092950aa7 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix) echo "derivation is $drvPath" -nix-store -q --tree "$drvPath" | grep '───.*builder1.sh' +nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' # Test Graphviz graph generation. nix-store -q --graph "$drvPath" > $TEST_ROOT/graph diff --git a/tests/export-graph.sh b/tests/export-graph.sh index a6fd69054..a1449b34e 100644 --- a/tests/export-graph.sh +++ b/tests/export-graph.sh @@ -11,7 +11,7 @@ checkRef() { outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result) -test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references" +test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references" checkRef input-2 for i in $(cat $outPath); do checkRef $i; done diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix index c0595cc47..21671ea2c 100644 --- a/tests/gc-concurrent.nix +++ b/tests/gc-concurrent.nix @@ -4,12 +4,12 @@ rec { input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; test1 = mkDerivation { @@ -23,5 +23,5 @@ rec { builder = ./gc-concurrent2.builder.sh; inherit input1 input2; }; - + } diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 93f837bef..49c68981a 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -32,10 +32,10 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies +nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles @@ -51,9 +51,9 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies +nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] From 268ecf5b3f3efd4e2a1fd78d088bdc69c27b6c7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 16:46:25 +0200 Subject: [PATCH 239/350] nix: Don't require --experimental-features=nix-command for some subcommands --- src/nix/main.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 5cf09c4f0..ef301580a 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -167,12 +167,15 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); - settings.requireExperimentalFeature("nix-command"); - initPlugins(); if (!args.command) args.showHelpAndExit(); + if (args.command->first != "repl" + && args.command->first != "doctor" + && args.command->first != "upgrade-nix") + settings.requireExperimentalFeature("nix-command"); + Finally f([]() { stopProgressBar(); }); startProgressBar(args.printBuildLogs); From ebc024df2287085d48ed6194aa756fd70c07f76c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 May 2020 17:25:27 +0200 Subject: [PATCH 240/350] Show hint how to enable experimental features --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a0a2d850e..bee94cbd8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -130,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%s' is disabled", name); + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } bool Settings::isWSL1() From ec870b9c853ad86fb1ccb482ca87802f1b155a2c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 10:52:26 -0600 Subject: [PATCH 241/350] new pos format for more errors --- src/libexpr/eval.cc | 11 ++++++++--- src/libexpr/nixexpr.hh | 6 +++++- src/libexpr/primops/fetchTree.cc | 23 +++++++++++++++++------ src/libexpr/primops/fromTOML.cc | 6 +++++- src/libstore/local-store.cc | 1 - 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 65071206f..7af665118 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -529,7 +529,12 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) { - throw EvalError(s, sym, p1, p2); + // p1 is where the error occurred; p2 is a position mentioned in the message. + throw EvalError( + ErrorInfo { + .hint = hintfmt(s, sym, p2), + .nixCode = NixCode { .errPos = p1 } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) @@ -638,7 +643,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!env->prevWith) - throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); + throwUndefinedVarError("undefined variable '%1%'", var.name, var.pos); for (size_t l = env->prevWith; l; --l, env = env->up) ; } } @@ -950,7 +955,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Symbol nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos); + throwEvalError("dynamic attribute '%1%' already defined at %2%", nameSym, i.pos, *j->pos); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index a185d5785..47d0e85ec 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -236,7 +236,11 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError("duplicate formal function argument '%1%' at %2%", arg, pos); + throw ParseError( + ErrorInfo { + .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .nixCode = NixCode { .errPos = pos } + }); }; void setName(Symbol & name); string showNamePos() const; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c5a0d9886..af3b61d6a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -66,7 +66,11 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V } if (!attrs.count("type")) - throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); + throw Error( + ErrorInfo { + .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .nixCode = NixCode { .errPos = pos } + }); input = fetchers::inputFromAttrs(attrs); } else @@ -107,13 +111,20 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to '%s', at %s", - attr.name, who, *attr.pos); - } + throw EvalError( + ErrorInfo { + .hint = hintfmt("unsupported argument '%s' to '%s'", + attr.name, who), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } if (!url) - throw EvalError("'url' argument required, at %s", pos); - + throw EvalError( + ErrorInfo { + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.forceStringNoCtx(*args[0], pos); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index c43324dbb..948069401 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -81,7 +81,11 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va try { visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { - throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + throw EvalError( + ErrorInfo { + .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .nixCode = NixCode { .errPos = pos } + }); } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eb7548543..1f16a22b1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -6,7 +6,6 @@ #include "derivations.hh" #include "nar-info.hh" #include "references.hh" -#include "error.hh" #include #include From 9e12b2f5b86963bc6d3a4ee37cc759357fb62263 Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Tue, 12 May 2020 18:58:13 +0200 Subject: [PATCH 242/350] Expose installer configuration environment variables via command line flags --- scripts/install-nix-from-closure.sh | 59 ++++++++++++++++++----------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 21915b37d..e06530ddf 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -40,30 +40,43 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then fi INSTALL_MODE=no-daemon -# Trivially handle the --daemon / --no-daemon options -if [ "x${1:-}" = "x--no-daemon" ]; then - INSTALL_MODE=no-daemon -elif [ "x${1:-}" = "x--daemon" ]; then - INSTALL_MODE=daemon -elif [ "x${1:-}" != "x" ]; then - ( - echo "Nix Installer [--daemon|--no-daemon]" - echo "Choose installation method." - echo "" - echo " --daemon: Installs and configures a background daemon that manages the store," - echo " providing multi-user support and better isolation for local builds." - echo " Both for security and reproducibility, this method is recommended if" - echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" - echo "" - echo " --no-daemon: Simple, single-user installation that does not require root and is" - echo " trivial to uninstall." - echo " (default)" - echo "" - ) >&2 - exit -fi +# handle the command line flags +while [ "x${1:-}" != "x" ]; do + if [ "x${1:-}" = "x--no-daemon" ]; then + INSTALL_MODE=no-daemon + elif [ "x${1:-}" = "x--daemon" ]; then + INSTALL_MODE=daemon + elif [ "x${1:-}" = "x--no-channel-add" ]; then + NIX_INSTALLER_NO_CHANNEL_ADD=1 + elif [ "x${1:-}" = "x--no-modify-profile" ]; then + NIX_INSTALLER_NO_MODIFY_PROFILE=1 + elif [ "x${1:-}" != "x" ]; then + ( + echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." + echo "" + echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" + echo " is installed by default." + echo "" + ) >&2 + exit + fi + shift +done if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' From 2a19bf86197083e17d9b35596c21e3c1b3f4a170 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 11:27:37 -0600 Subject: [PATCH 243/350] move pos to the first arg, to indicate its not used in a fmt template --- src/libexpr/eval-inline.hh | 10 +++---- src/libexpr/eval.cc | 58 +++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index e6b838665..eee49e02e 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -7,7 +7,7 @@ namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { throw EvalError( ErrorInfo { @@ -22,7 +22,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { throw TypeError( ErrorInfo { @@ -51,7 +51,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) else if (v.type == tApp) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered", pos); + throwEvalError(pos, "infinite recursion encountered"); } @@ -67,7 +67,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected", v, pos); + throwTypeError(pos, "value is %1% while a set was expected", v); } @@ -83,7 +83,7 @@ inline void EvalState::forceList(Value & v, const Pos & pos) { forceValue(v, pos); if (!v.isList()) - throwTypeError("value is %1% while a list was expected", v, pos); + throwTypeError(pos, "value is %1% while a list was expected", v); } /* Note: Various places expect the allocated memory to be zeroed. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7af665118..e4bc6dc83 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -504,7 +504,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) throw EvalError(s, s2); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { throw EvalError( ErrorInfo { @@ -518,7 +518,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con throw EvalError(s, s2, s3); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { throw EvalError( ErrorInfo { @@ -527,7 +527,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con }); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2)) { // p1 is where the error occurred; p2 is a position mentioned in the message. throw EvalError( @@ -537,7 +537,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, co }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { throw TypeError( ErrorInfo { @@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) throw TypeError(s, s1); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError( ErrorInfo { @@ -560,7 +560,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun }); } -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) +LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { throw AssertionError( ErrorInfo { @@ -569,7 +569,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1 }); } -LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) +LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { throw UndefinedVarError( ErrorInfo { @@ -643,7 +643,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!env->prevWith) - throwUndefinedVarError("undefined variable '%1%'", var.name, var.pos); + throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name); for (size_t l = env->prevWith; l; --l, env = env->up) ; } } @@ -841,7 +841,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) Value v; e->eval(*this, env, v); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -955,7 +955,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Symbol nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError("dynamic attribute '%1%' already defined at %2%", nameSym, i.pos, *j->pos); + throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -1043,7 +1043,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute '%1%' missing", name, pos); + throwEvalError(pos, "attribute '%1%' missing", name); } vAttrs = j->value; pos2 = j->pos; @@ -1169,7 +1169,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%", fun, pos); + throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1197,8 +1197,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po for (auto & i : lambda.formals->formals) { Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { - if (!i.def) throwTypeError("%1% called without required argument '%2%'", - lambda, i.name, pos); + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1213,7 +1213,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po user. */ for (auto & i : *arg.attrs) if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument '%2%'", lambda, i.name, pos); + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); abort(); // can't happen } } @@ -1302,7 +1302,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos); + throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); } body->eval(state, env, v); } @@ -1454,14 +1454,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to an integer", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == tFloat) { if (vTmp.type == tInt) { nf += vTmp.integer; } else if (vTmp.type == tFloat) { nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to a float", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else s << state.coerceToString(pos, vTmp, context, false, firstType == tString); } @@ -1472,7 +1472,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) mkFloat(v, nf); else if (firstType == tPath) { if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path", pos); + throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); mkPath(v, path.c_str()); } else @@ -1521,7 +1521,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected", v, pos); + throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1532,7 +1532,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) if (v.type == tInt) return v.integer; else if (v.type != tFloat) - throwTypeError("value is %1% while a float was expected", v, pos); + throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1541,7 +1541,7 @@ bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1556,7 +1556,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) - throwTypeError("value is %1% while a function was expected", v, pos); + throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1565,7 +1565,7 @@ string EvalState::forceString(Value & v, const Pos & pos) forceValue(v, pos); if (v.type != tString) { if (pos) - throwTypeError("value is %1% while a string was expected", v, pos); + throwTypeError(pos, "value is %1% while a string was expected", v); else throwTypeError("value is %1% while a string was expected", v); } @@ -1594,8 +1594,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) string s = forceString(v, pos); if (v.string.context) { if (pos) - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0], pos); + throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string.s, v.string.context[0]); else throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]); @@ -1651,7 +1651,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return *maybeString; } auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string", pos); + if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } @@ -1682,7 +1682,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } } - throwTypeError("cannot coerce %1% to a string", v, pos); + throwTypeError(pos, "cannot coerce %1% to a string", v); } @@ -1713,7 +1713,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { string path = coerceToString(pos, v, context, false, false); if (path == "" || path[0] != '/') - throwEvalError("string '%1%' doesn't represent an absolute path", path, pos); + throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); return path; } From 19cffc29c995af8fb155f2764664f0b079879b4f Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 12:09:12 -0600 Subject: [PATCH 244/350] remove unused extra json fields --- src/libstore/local-store.cc | 1 - src/libstore/remote-store.cc | 9 --------- src/libutil/logging.cc | 24 ------------------------ 3 files changed, 34 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1f16a22b1..63a825af0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1364,7 +1364,6 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - // TODO log as warning if repair successful?? logError( ErrorInfo { .name = "Missing path with referrers", diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7c50c1065..cc336e460 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -780,15 +780,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_NEXT) - // TODO: is this really an ErrorInfo error? Seems like we're forwarding the - // stderr output of the remote to current stderr/log - // ErrorInfo gets lost in this scenario. - // An alternative might be a logger on the remote that forwards ErrorInfo and etc. - // logError( - // ErrorInfo { - // // .name = "Remote Store" TODO reasonable name. - // .hint = hintfmt(chomp(readString(from))) - // }); printError(chomp(readString(from))); else if (msg == STDERR_START_ACTIVITY) { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7c75456ee..736ce604e 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -150,7 +150,6 @@ struct JSONLogger : Logger { void logEI(const ErrorInfo & ei) override { - // add fields like Pos info and etc? std::ostringstream oss; oss << ei; @@ -159,29 +158,6 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); - // Extra things that COULD go into json. Useful? - // TODO: decide if useful. - // TODO: make a json obj that goes into json["msg"]? - json["name"] = ei.name; - json["description"] = ei.description; - if (ei.hint.has_value()) { - json["hint"] = ei.hint->str(); - } - if (ei.nixCode.has_value()) { - if (ei.nixCode->errPos.line != 0) - json["line"] = ei.nixCode->errPos.line; - if (ei.nixCode->errPos.column != 0) - json["column"] = ei.nixCode->errPos.column; - if (ei.nixCode->errPos.file != "") - json["file"] = ei.nixCode->errPos.file; - if (ei.nixCode->prevLineOfCode.has_value()) - json["prevLineOfCode"] = *ei.nixCode->prevLineOfCode; - if (ei.nixCode->errLineOfCode.has_value()) - json["errLineOfCode"] = *ei.nixCode->errLineOfCode; - if (ei.nixCode->nextLineOfCode.has_value()) - json["nextLineOfCode"] = *ei.nixCode->nextLineOfCode; - } - write(json); } From d608793e4feb79fe615e3071c7ce561dd2dea0a2 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 12:09:57 -0600 Subject: [PATCH 245/350] remove uncrustify cfg --- .uncrustify.cfg | 2990 ----------------------------------------------- 1 file changed, 2990 deletions(-) delete mode 100644 .uncrustify.cfg diff --git a/.uncrustify.cfg b/.uncrustify.cfg deleted file mode 100644 index 6bf91ec3c..000000000 --- a/.uncrustify.cfg +++ /dev/null @@ -1,2990 +0,0 @@ -# Uncrustify-0.70.1_f - -# -# General options -# - -# The type of line endings. -# -# Default: auto -newlines = auto # lf/crlf/cr/auto - -# The original size of tabs in the input. -# -# Default: 8 -input_tab_size = 4 # unsigned number - -# The size of tabs in the output (only used if align_with_tabs=true). -# -# Default: 8 -output_tab_size = 4 # unsigned number - -# The ASCII value of the string escape char, usually 92 (\) or (Pawn) 94 (^). -# -# Default: 92 -string_escape_char = 92 # unsigned number - -# Alternate string escape char (usually only used for Pawn). -# Only works right before the quote char. -string_escape_char2 = 0 # unsigned number - -# Replace tab characters found in string literals with the escape sequence \t -# instead. -string_replace_tab_chars = false # true/false - -# Allow interpreting '>=' and '>>=' as part of a template in code like -# 'void f(list>=val);'. If true, 'assert(x<0 && y>=3)' will be broken. -# Improvements to template detection may make this option obsolete. -tok_split_gte = false # true/false - -# Specify the marker used in comments to disable processing of part of the -# file. -# -# Default: *INDENT-OFF* -disable_processing_cmt = " *INDENT-OFF*" # string - -# Specify the marker used in comments to (re)enable processing in a file. -# -# Default: *INDENT-ON* -enable_processing_cmt = " *INDENT-ON*" # string - -# Enable parsing of digraphs. -enable_digraphs = false # true/false - -# Add or remove the UTF-8 BOM (recommend 'remove'). -utf8_bom = ignore # ignore/add/remove/force - -# If the file contains bytes with values between 128 and 255, but is not -# UTF-8, then output as UTF-8. -utf8_byte = false # true/false - -# Force the output encoding to UTF-8. -utf8_force = false # true/false - -# Add or remove space between 'do' and '{'. -sp_do_brace_open = ignore # ignore/add/remove/force - -# Add or remove space between '}' and 'while'. -sp_brace_close_while = ignore # ignore/add/remove/force - -# Add or remove space between 'while' and '('. -sp_while_paren_open = ignore # ignore/add/remove/force - -# -# Spacing options -# - -# Add or remove space around non-assignment symbolic operators ('+', '/', '%', -# '<<', and so forth). -sp_arith = ignore # ignore/add/remove/force - -# Add or remove space around arithmetic operators '+' and '-'. -# -# Overrides sp_arith. -sp_arith_additive = ignore # ignore/add/remove/force - -# Add or remove space around assignment operator '=', '+=', etc. -sp_assign = ignore # ignore/add/remove/force - -# Add or remove space around '=' in C++11 lambda capture specifications. -# -# Overrides sp_assign. -sp_cpp_lambda_assign = ignore # ignore/add/remove/force - -# Add or remove space after the capture specification of a C++11 lambda when -# an argument list is present, as in '[] (int x){ ... }'. -sp_cpp_lambda_square_paren = ignore # ignore/add/remove/force - -# Add or remove space after the capture specification of a C++11 lambda with -# no argument list is present, as in '[] { ... }'. -sp_cpp_lambda_square_brace = remove # ignore/add/remove/force - -# Add or remove space after the argument list of a C++11 lambda, as in -# '[](int x) { ... }'. -sp_cpp_lambda_paren_brace = ignore # ignore/add/remove/force - -# Add or remove space between a lambda body and its call operator of an -# immediately invoked lambda, as in '[]( ... ){ ... } ( ... )'. -sp_cpp_lambda_fparen = ignore # ignore/add/remove/force - -# Add or remove space around assignment operator '=' in a prototype. -# -# If set to ignore, use sp_assign. -sp_assign_default = ignore # ignore/add/remove/force - -# Add or remove space before assignment operator '=', '+=', etc. -# -# Overrides sp_assign. -sp_before_assign = ignore # ignore/add/remove/force - -# Add or remove space after assignment operator '=', '+=', etc. -# -# Overrides sp_assign. -sp_after_assign = ignore # ignore/add/remove/force - -# Add or remove space in 'NS_ENUM ('. -sp_enum_paren = ignore # ignore/add/remove/force - -# Add or remove space around assignment '=' in enum. -sp_enum_assign = ignore # ignore/add/remove/force - -# Add or remove space before assignment '=' in enum. -# -# Overrides sp_enum_assign. -sp_enum_before_assign = ignore # ignore/add/remove/force - -# Add or remove space after assignment '=' in enum. -# -# Overrides sp_enum_assign. -sp_enum_after_assign = ignore # ignore/add/remove/force - -# Add or remove space around assignment ':' in enum. -sp_enum_colon = ignore # ignore/add/remove/force - -# Add or remove space around preprocessor '##' concatenation operator. -# -# Default: add -sp_pp_concat = add # ignore/add/remove/force - -# Add or remove space after preprocessor '#' stringify operator. -# Also affects the '#@' charizing operator. -sp_pp_stringify = ignore # ignore/add/remove/force - -# Add or remove space before preprocessor '#' stringify operator -# as in '#define x(y) L#y'. -sp_before_pp_stringify = ignore # ignore/add/remove/force - -# Add or remove space around boolean operators '&&' and '||'. -sp_bool = ignore # ignore/add/remove/force - -# Add or remove space around compare operator '<', '>', '==', etc. -sp_compare = ignore # ignore/add/remove/force - -# Add or remove space inside '(' and ')'. -sp_inside_paren = ignore # ignore/add/remove/force - -# Add or remove space between nested parentheses, i.e. '((' vs. ') )'. -sp_paren_paren = ignore # ignore/add/remove/force - -# Add or remove space between back-to-back parentheses, i.e. ')(' vs. ') ('. -sp_cparen_oparen = ignore # ignore/add/remove/force - -# Whether to balance spaces inside nested parentheses. -sp_balance_nested_parens = false # true/false - -# Add or remove space between ')' and '{'. -sp_paren_brace = ignore # ignore/add/remove/force - -# Add or remove space between nested braces, i.e. '{{' vs '{ {'. -sp_brace_brace = ignore # ignore/add/remove/force - -# Add or remove space before pointer star '*'. -sp_before_ptr_star = ignore # ignore/add/remove/force - -# Add or remove space before pointer star '*' that isn't followed by a -# variable name. If set to ignore, sp_before_ptr_star is used instead. -sp_before_unnamed_ptr_star = ignore # ignore/add/remove/force - -# Add or remove space between pointer stars '*'. -sp_between_ptr_star = ignore # ignore/add/remove/force - -# Add or remove space after pointer star '*', if followed by a word. -# -# Overrides sp_type_func. -sp_after_ptr_star = ignore # ignore/add/remove/force - -# Add or remove space after pointer caret '^', if followed by a word. -sp_after_ptr_block_caret = ignore # ignore/add/remove/force - -# Add or remove space after pointer star '*', if followed by a qualifier. -sp_after_ptr_star_qualifier = ignore # ignore/add/remove/force - -# Add or remove space after a pointer star '*', if followed by a function -# prototype or function definition. -# -# Overrides sp_after_ptr_star and sp_type_func. -sp_after_ptr_star_func = ignore # ignore/add/remove/force - -# Add or remove space after a pointer star '*', if followed by an open -# parenthesis, as in 'void* (*)(). -sp_ptr_star_paren = ignore # ignore/add/remove/force - -# Add or remove space before a pointer star '*', if followed by a function -# prototype or function definition. -sp_before_ptr_star_func = ignore # ignore/add/remove/force - -# Add or remove space before a reference sign '&'. -sp_before_byref = ignore # ignore/add/remove/force - -# Add or remove space before a reference sign '&' that isn't followed by a -# variable name. If set to ignore, sp_before_byref is used instead. -sp_before_unnamed_byref = ignore # ignore/add/remove/force - -# Add or remove space after reference sign '&', if followed by a word. -# -# Overrides sp_type_func. -sp_after_byref = ignore # ignore/add/remove/force - -# Add or remove space after a reference sign '&', if followed by a function -# prototype or function definition. -# -# Overrides sp_after_byref and sp_type_func. -sp_after_byref_func = ignore # ignore/add/remove/force - -# Add or remove space before a reference sign '&', if followed by a function -# prototype or function definition. -sp_before_byref_func = ignore # ignore/add/remove/force - -# Add or remove space between type and word. -# -# Default: force -sp_after_type = force # ignore/add/remove/force - -# Add or remove space between 'decltype(...)' and word. -sp_after_decltype = ignore # ignore/add/remove/force - -# (D) Add or remove space before the parenthesis in the D constructs -# 'template Foo(' and 'class Foo('. -sp_before_template_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'template' and '<'. -# If set to ignore, sp_before_angle is used. -sp_template_angle = ignore # ignore/add/remove/force - -# Add or remove space before '<'. -sp_before_angle = ignore # ignore/add/remove/force - -# Add or remove space inside '<' and '>'. -sp_inside_angle = ignore # ignore/add/remove/force - -# Add or remove space inside '<>'. -sp_inside_angle_empty = ignore # ignore/add/remove/force - -# Add or remove space between '>' and ':'. -sp_angle_colon = ignore # ignore/add/remove/force - -# Add or remove space after '>'. -sp_after_angle = ignore # ignore/add/remove/force - -# Add or remove space between '>' and '(' as found in 'new List(foo);'. -sp_angle_paren = ignore # ignore/add/remove/force - -# Add or remove space between '>' and '()' as found in 'new List();'. -sp_angle_paren_empty = ignore # ignore/add/remove/force - -# Add or remove space between '>' and a word as in 'List m;' or -# 'template static ...'. -sp_angle_word = ignore # ignore/add/remove/force - -# Add or remove space between '>' and '>' in '>>' (template stuff). -# -# Default: add -sp_angle_shift = ignore # ignore/add/remove/force - -# (C++11) Permit removal of the space between '>>' in 'foo >'. Note -# that sp_angle_shift cannot remove the space without this option. -sp_permit_cpp11_shift = true # true/false - -# Add or remove space before '(' of control statements ('if', 'for', 'switch', -# 'while', etc.). -sp_before_sparen = ignore # ignore/add/remove/force - -# Add or remove space inside '(' and ')' of control statements. -sp_inside_sparen = ignore # ignore/add/remove/force - -# Add or remove space after '(' of control statements. -# -# Overrides sp_inside_sparen. -sp_inside_sparen_open = ignore # ignore/add/remove/force - -# Add or remove space before ')' of control statements. -# -# Overrides sp_inside_sparen. -sp_inside_sparen_close = ignore # ignore/add/remove/force - -# Add or remove space after ')' of control statements. -sp_after_sparen = ignore # ignore/add/remove/force - -# Add or remove space between ')' and '{' of of control statements. -sp_sparen_brace = ignore # ignore/add/remove/force - -# (D) Add or remove space between 'invariant' and '('. -sp_invariant_paren = ignore # ignore/add/remove/force - -# (D) Add or remove space after the ')' in 'invariant (C) c'. -sp_after_invariant_paren = ignore # ignore/add/remove/force - -# Add or remove space before empty statement ';' on 'if', 'for' and 'while'. -sp_special_semi = ignore # ignore/add/remove/force - -# Add or remove space before ';'. -# -# Default: remove -sp_before_semi = remove # ignore/add/remove/force - -# Add or remove space before ';' in non-empty 'for' statements. -sp_before_semi_for = ignore # ignore/add/remove/force - -# Add or remove space before a semicolon of an empty part of a for statement. -sp_before_semi_for_empty = ignore # ignore/add/remove/force - -# Add or remove space after ';', except when followed by a comment. -# -# Default: add -sp_after_semi = add # ignore/add/remove/force - -# Add or remove space after ';' in non-empty 'for' statements. -# -# Default: force -sp_after_semi_for = force # ignore/add/remove/force - -# Add or remove space after the final semicolon of an empty part of a for -# statement, as in 'for ( ; ; )'. -sp_after_semi_for_empty = ignore # ignore/add/remove/force - -# Add or remove space before '[' (except '[]'). -sp_before_square = ignore # ignore/add/remove/force - -# Add or remove space before '[' for a variable definition. -# -# Default: remove -sp_before_vardef_square = remove # ignore/add/remove/force - -# Add or remove space before '[' for asm block. -sp_before_square_asm_block = ignore # ignore/add/remove/force - -# Add or remove space before '[]'. -sp_before_squares = ignore # ignore/add/remove/force - -# Add or remove space before C++17 structured bindings. -sp_cpp_before_struct_binding = ignore # ignore/add/remove/force - -# Add or remove space inside a non-empty '[' and ']'. -sp_inside_square = ignore # ignore/add/remove/force - -# (OC) Add or remove space inside a non-empty Objective-C boxed array '@[' and -# ']'. If set to ignore, sp_inside_square is used. -sp_inside_square_oc_array = ignore # ignore/add/remove/force - -# Add or remove space after ',', i.e. 'a,b' vs. 'a, b'. -sp_after_comma = ignore # ignore/add/remove/force - -# Add or remove space before ','. -# -# Default: remove -sp_before_comma = remove # ignore/add/remove/force - -# (C#) Add or remove space between ',' and ']' in multidimensional array type -# like 'int[,,]'. -sp_after_mdatype_commas = ignore # ignore/add/remove/force - -# (C#) Add or remove space between '[' and ',' in multidimensional array type -# like 'int[,,]'. -sp_before_mdatype_commas = ignore # ignore/add/remove/force - -# (C#) Add or remove space between ',' in multidimensional array type -# like 'int[,,]'. -sp_between_mdatype_commas = ignore # ignore/add/remove/force - -# Add or remove space between an open parenthesis and comma, -# i.e. '(,' vs. '( ,'. -# -# Default: force -sp_paren_comma = force # ignore/add/remove/force - -# Add or remove space before the variadic '...' when preceded by a -# non-punctuator. -sp_before_ellipsis = remove # ignore/add/remove/force - -# Add or remove space between a type and '...'. -sp_type_ellipsis = ignore # ignore/add/remove/force - -# (D) Add or remove space between a type and '?'. -sp_type_question = ignore # ignore/add/remove/force - -# Add or remove space between ')' and '...'. -sp_paren_ellipsis = ignore # ignore/add/remove/force - -# Add or remove space between ')' and a qualifier such as 'const'. -sp_paren_qualifier = ignore # ignore/add/remove/force - -# Add or remove space between ')' and 'noexcept'. -sp_paren_noexcept = ignore # ignore/add/remove/force - -# Add or remove space after class ':'. -sp_after_class_colon = ignore # ignore/add/remove/force - -# Add or remove space before class ':'. -sp_before_class_colon = ignore # ignore/add/remove/force - -# Add or remove space after class constructor ':'. -sp_after_constr_colon = ignore # ignore/add/remove/force - -# Add or remove space before class constructor ':'. -sp_before_constr_colon = ignore # ignore/add/remove/force - -# Add or remove space before case ':'. -# -# Default: remove -sp_before_case_colon = remove # ignore/add/remove/force - -# Add or remove space between 'operator' and operator sign. -sp_after_operator = ignore # ignore/add/remove/force - -# Add or remove space between the operator symbol and the open parenthesis, as -# in 'operator ++('. -sp_after_operator_sym = ignore # ignore/add/remove/force - -# Overrides sp_after_operator_sym when the operator has no arguments, as in -# 'operator *()'. -sp_after_operator_sym_empty = ignore # ignore/add/remove/force - -# Add or remove space after C/D cast, i.e. 'cast(int)a' vs. 'cast(int) a' or -# '(int)a' vs. '(int) a'. -sp_after_cast = ignore # ignore/add/remove/force - -# Add or remove spaces inside cast parentheses. -sp_inside_paren_cast = ignore # ignore/add/remove/force - -# Add or remove space between the type and open parenthesis in a C++ cast, -# i.e. 'int(exp)' vs. 'int (exp)'. -sp_cpp_cast_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'sizeof' and '('. -sp_sizeof_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'sizeof' and '...'. -sp_sizeof_ellipsis = ignore # ignore/add/remove/force - -# Add or remove space between 'sizeof...' and '('. -sp_sizeof_ellipsis_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'decltype' and '('. -sp_decltype_paren = ignore # ignore/add/remove/force - -# (Pawn) Add or remove space after the tag keyword. -sp_after_tag = ignore # ignore/add/remove/force - -# Add or remove space inside enum '{' and '}'. -sp_inside_braces_enum = ignore # ignore/add/remove/force - -# Add or remove space inside struct/union '{' and '}'. -sp_inside_braces_struct = ignore # ignore/add/remove/force - -# (OC) Add or remove space inside Objective-C boxed dictionary '{' and '}' -sp_inside_braces_oc_dict = ignore # ignore/add/remove/force - -# Add or remove space after open brace in an unnamed temporary -# direct-list-initialization. -sp_after_type_brace_init_lst_open = ignore # ignore/add/remove/force - -# Add or remove space before close brace in an unnamed temporary -# direct-list-initialization. -sp_before_type_brace_init_lst_close = ignore # ignore/add/remove/force - -# Add or remove space inside an unnamed temporary direct-list-initialization. -sp_inside_type_brace_init_lst = ignore # ignore/add/remove/force - -# Add or remove space inside '{' and '}'. -sp_inside_braces = ignore # ignore/add/remove/force - -# Add or remove space inside '{}'. -sp_inside_braces_empty = ignore # ignore/add/remove/force - -# Add or remove space around trailing return operator '->'. -sp_trailing_return = ignore # ignore/add/remove/force - -# Add or remove space between return type and function name. A minimum of 1 -# is forced except for pointer return types. -sp_type_func = ignore # ignore/add/remove/force - -# Add or remove space between type and open brace of an unnamed temporary -# direct-list-initialization. -sp_type_brace_init_lst = ignore # ignore/add/remove/force - -# Add or remove space between function name and '(' on function declaration. -sp_func_proto_paren = ignore # ignore/add/remove/force - -# Add or remove space between function name and '()' on function declaration -# without parameters. -sp_func_proto_paren_empty = ignore # ignore/add/remove/force - -# Add or remove space between function name and '(' with a typedef specifier. -sp_func_type_paren = ignore # ignore/add/remove/force - -# Add or remove space between alias name and '(' of a non-pointer function type typedef. -sp_func_def_paren = ignore # ignore/add/remove/force - -# Add or remove space between function name and '()' on function definition -# without parameters. -sp_func_def_paren_empty = ignore # ignore/add/remove/force - -# Add or remove space inside empty function '()'. -# Overrides sp_after_angle unless use_sp_after_angle_always is set to true. -sp_inside_fparens = ignore # ignore/add/remove/force - -# Add or remove space inside function '(' and ')'. -sp_inside_fparen = ignore # ignore/add/remove/force - -# Add or remove space inside the first parentheses in a function type, as in -# 'void (*x)(...)'. -sp_inside_tparen = ignore # ignore/add/remove/force - -# Add or remove space between the ')' and '(' in a function type, as in -# 'void (*x)(...)'. -sp_after_tparen_close = ignore # ignore/add/remove/force - -# Add or remove space between ']' and '(' when part of a function call. -sp_square_fparen = ignore # ignore/add/remove/force - -# Add or remove space between ')' and '{' of function. -sp_fparen_brace = ignore # ignore/add/remove/force - -# Add or remove space between ')' and '{' of s function call in object -# initialization. -# -# Overrides sp_fparen_brace. -sp_fparen_brace_initializer = ignore # ignore/add/remove/force - -# (Java) Add or remove space between ')' and '{{' of double brace initializer. -sp_fparen_dbrace = ignore # ignore/add/remove/force - -# Add or remove space between function name and '(' on function calls. -sp_func_call_paren = ignore # ignore/add/remove/force - -# Add or remove space between function name and '()' on function calls without -# parameters. If set to ignore (the default), sp_func_call_paren is used. -sp_func_call_paren_empty = ignore # ignore/add/remove/force - -# Add or remove space between the user function name and '(' on function -# calls. You need to set a keyword to be a user function in the config file, -# like: -# set func_call_user tr _ i18n -sp_func_call_user_paren = ignore # ignore/add/remove/force - -# Add or remove space inside user function '(' and ')'. -sp_func_call_user_inside_fparen = ignore # ignore/add/remove/force - -# Add or remove space between nested parentheses with user functions, -# i.e. '((' vs. '( ('. -sp_func_call_user_paren_paren = ignore # ignore/add/remove/force - -# Add or remove space between a constructor/destructor and the open -# parenthesis. -sp_func_class_paren = ignore # ignore/add/remove/force - -# Add or remove space between a constructor without parameters or destructor -# and '()'. -sp_func_class_paren_empty = ignore # ignore/add/remove/force - -# Add or remove space between 'return' and '('. -sp_return_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'return' and '{'. -sp_return_brace = ignore # ignore/add/remove/force - -# Add or remove space between '__attribute__' and '('. -sp_attribute_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'defined' and '(' in '#if defined (FOO)'. -sp_defined_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'throw' and '(' in 'throw (something)'. -sp_throw_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'throw' and anything other than '(' as in -# '@throw [...];'. -sp_after_throw = ignore # ignore/add/remove/force - -# Add or remove space between 'catch' and '(' in 'catch (something) { }'. -# If set to ignore, sp_before_sparen is used. -sp_catch_paren = ignore # ignore/add/remove/force - -# (OC) Add or remove space between '@catch' and '(' -# in '@catch (something) { }'. If set to ignore, sp_catch_paren is used. -sp_oc_catch_paren = ignore # ignore/add/remove/force - -# (OC) Add or remove space between class name and '(' -# in '@interface className(categoryName):BaseClass' -sp_oc_classname_paren = ignore # ignore/add/remove/force - -# (D) Add or remove space between 'version' and '(' -# in 'version (something) { }'. If set to ignore, sp_before_sparen is used. -sp_version_paren = ignore # ignore/add/remove/force - -# (D) Add or remove space between 'scope' and '(' -# in 'scope (something) { }'. If set to ignore, sp_before_sparen is used. -sp_scope_paren = ignore # ignore/add/remove/force - -# Add or remove space between 'super' and '(' in 'super (something)'. -# -# Default: remove -sp_super_paren = remove # ignore/add/remove/force - -# Add or remove space between 'this' and '(' in 'this (something)'. -# -# Default: remove -sp_this_paren = remove # ignore/add/remove/force - -# Add or remove space between a macro name and its definition. -sp_macro = ignore # ignore/add/remove/force - -# Add or remove space between a macro function ')' and its definition. -sp_macro_func = ignore # ignore/add/remove/force - -# Add or remove space between 'else' and '{' if on the same line. -sp_else_brace = ignore # ignore/add/remove/force - -# Add or remove space between '}' and 'else' if on the same line. -sp_brace_else = ignore # ignore/add/remove/force - -# Add or remove space between '}' and the name of a typedef on the same line. -sp_brace_typedef = ignore # ignore/add/remove/force - -# Add or remove space before the '{' of a 'catch' statement, if the '{' and -# 'catch' are on the same line, as in 'catch (decl) {'. -sp_catch_brace = ignore # ignore/add/remove/force - -# (OC) Add or remove space before the '{' of a '@catch' statement, if the '{' -# and '@catch' are on the same line, as in '@catch (decl) {'. -# If set to ignore, sp_catch_brace is used. -sp_oc_catch_brace = ignore # ignore/add/remove/force - -# Add or remove space between '}' and 'catch' if on the same line. -sp_brace_catch = ignore # ignore/add/remove/force - -# (OC) Add or remove space between '}' and '@catch' if on the same line. -# If set to ignore, sp_brace_catch is used. -sp_oc_brace_catch = ignore # ignore/add/remove/force - -# Add or remove space between 'finally' and '{' if on the same line. -sp_finally_brace = ignore # ignore/add/remove/force - -# Add or remove space between '}' and 'finally' if on the same line. -sp_brace_finally = ignore # ignore/add/remove/force - -# Add or remove space between 'try' and '{' if on the same line. -sp_try_brace = ignore # ignore/add/remove/force - -# Add or remove space between get/set and '{' if on the same line. -sp_getset_brace = ignore # ignore/add/remove/force - -# Add or remove space between a variable and '{' for C++ uniform -# initialization. -# -# Default: add -sp_word_brace = add # ignore/add/remove/force - -# Add or remove space between a variable and '{' for a namespace. -# -# Default: add -sp_word_brace_ns = add # ignore/add/remove/force - -# Add or remove space before the '::' operator. -sp_before_dc = ignore # ignore/add/remove/force - -# Add or remove space after the '::' operator. -sp_after_dc = ignore # ignore/add/remove/force - -# (D) Add or remove around the D named array initializer ':' operator. -sp_d_array_colon = ignore # ignore/add/remove/force - -# Add or remove space after the '!' (not) unary operator. -# -# Default: remove -sp_not = remove # ignore/add/remove/force - -# Add or remove space after the '~' (invert) unary operator. -# -# Default: remove -sp_inv = remove # ignore/add/remove/force - -# Add or remove space after the '&' (address-of) unary operator. This does not -# affect the spacing after a '&' that is part of a type. -# -# Default: remove -sp_addr = remove # ignore/add/remove/force - -# Add or remove space around the '.' or '->' operators. -# -# Default: remove -sp_member = remove # ignore/add/remove/force - -# Add or remove space after the '*' (dereference) unary operator. This does -# not affect the spacing after a '*' that is part of a type. -# -# Default: remove -sp_deref = remove # ignore/add/remove/force - -# Add or remove space after '+' or '-', as in 'x = -5' or 'y = +7'. -# -# Default: remove -sp_sign = remove # ignore/add/remove/force - -# Add or remove space between '++' and '--' the word to which it is being -# applied, as in '(--x)' or 'y++;'. -# -# Default: remove -sp_incdec = remove # ignore/add/remove/force - -# Add or remove space before a backslash-newline at the end of a line. -# -# Default: add -sp_before_nl_cont = add # ignore/add/remove/force - -# (OC) Add or remove space after the scope '+' or '-', as in '-(void) foo;' -# or '+(int) bar;'. -sp_after_oc_scope = ignore # ignore/add/remove/force - -# (OC) Add or remove space after the colon in message specs, -# i.e. '-(int) f:(int) x;' vs. '-(int) f: (int) x;'. -sp_after_oc_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space before the colon in message specs, -# i.e. '-(int) f: (int) x;' vs. '-(int) f : (int) x;'. -sp_before_oc_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space after the colon in immutable dictionary expression -# 'NSDictionary *test = @{@"foo" :@"bar"};'. -sp_after_oc_dict_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space before the colon in immutable dictionary expression -# 'NSDictionary *test = @{@"foo" :@"bar"};'. -sp_before_oc_dict_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space after the colon in message specs, -# i.e. '[object setValue:1];' vs. '[object setValue: 1];'. -sp_after_send_oc_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space before the colon in message specs, -# i.e. '[object setValue:1];' vs. '[object setValue :1];'. -sp_before_send_oc_colon = ignore # ignore/add/remove/force - -# (OC) Add or remove space after the (type) in message specs, -# i.e. '-(int)f: (int) x;' vs. '-(int)f: (int)x;'. -sp_after_oc_type = ignore # ignore/add/remove/force - -# (OC) Add or remove space after the first (type) in message specs, -# i.e. '-(int) f:(int)x;' vs. '-(int)f:(int)x;'. -sp_after_oc_return_type = ignore # ignore/add/remove/force - -# (OC) Add or remove space between '@selector' and '(', -# i.e. '@selector(msgName)' vs. '@selector (msgName)'. -# Also applies to '@protocol()' constructs. -sp_after_oc_at_sel = ignore # ignore/add/remove/force - -# (OC) Add or remove space between '@selector(x)' and the following word, -# i.e. '@selector(foo) a:' vs. '@selector(foo)a:'. -sp_after_oc_at_sel_parens = ignore # ignore/add/remove/force - -# (OC) Add or remove space inside '@selector' parentheses, -# i.e. '@selector(foo)' vs. '@selector( foo )'. -# Also applies to '@protocol()' constructs. -sp_inside_oc_at_sel_parens = ignore # ignore/add/remove/force - -# (OC) Add or remove space before a block pointer caret, -# i.e. '^int (int arg){...}' vs. ' ^int (int arg){...}'. -sp_before_oc_block_caret = ignore # ignore/add/remove/force - -# (OC) Add or remove space after a block pointer caret, -# i.e. '^int (int arg){...}' vs. '^ int (int arg){...}'. -sp_after_oc_block_caret = ignore # ignore/add/remove/force - -# (OC) Add or remove space between the receiver and selector in a message, -# as in '[receiver selector ...]'. -sp_after_oc_msg_receiver = ignore # ignore/add/remove/force - -# (OC) Add or remove space after '@property'. -sp_after_oc_property = ignore # ignore/add/remove/force - -# (OC) Add or remove space between '@synchronized' and the open parenthesis, -# i.e. '@synchronized(foo)' vs. '@synchronized (foo)'. -sp_after_oc_synchronized = ignore # ignore/add/remove/force - -# Add or remove space around the ':' in 'b ? t : f'. -sp_cond_colon = ignore # ignore/add/remove/force - -# Add or remove space before the ':' in 'b ? t : f'. -# -# Overrides sp_cond_colon. -sp_cond_colon_before = ignore # ignore/add/remove/force - -# Add or remove space after the ':' in 'b ? t : f'. -# -# Overrides sp_cond_colon. -sp_cond_colon_after = ignore # ignore/add/remove/force - -# Add or remove space around the '?' in 'b ? t : f'. -sp_cond_question = ignore # ignore/add/remove/force - -# Add or remove space before the '?' in 'b ? t : f'. -# -# Overrides sp_cond_question. -sp_cond_question_before = ignore # ignore/add/remove/force - -# Add or remove space after the '?' in 'b ? t : f'. -# -# Overrides sp_cond_question. -sp_cond_question_after = ignore # ignore/add/remove/force - -# In the abbreviated ternary form '(a ?: b)', add or remove space between '?' -# and ':'. -# -# Overrides all other sp_cond_* options. -sp_cond_ternary_short = ignore # ignore/add/remove/force - -# Fix the spacing between 'case' and the label. Only 'ignore' and 'force' make -# sense here. -sp_case_label = ignore # ignore/add/remove/force - -# (D) Add or remove space around the D '..' operator. -sp_range = ignore # ignore/add/remove/force - -# Add or remove space after ':' in a Java/C++11 range-based 'for', -# as in 'for (Type var : expr)'. -sp_after_for_colon = ignore # ignore/add/remove/force - -# Add or remove space before ':' in a Java/C++11 range-based 'for', -# as in 'for (Type var : expr)'. -sp_before_for_colon = ignore # ignore/add/remove/force - -# (D) Add or remove space between 'extern' and '(' as in 'extern (C)'. -sp_extern_paren = ignore # ignore/add/remove/force - -# Add or remove space after the opening of a C++ comment, -# i.e. '// A' vs. '//A'. -sp_cmt_cpp_start = ignore # ignore/add/remove/force - -# If true, space is added with sp_cmt_cpp_start will be added after doxygen -# sequences like '///', '///<', '//!' and '//!<'. -sp_cmt_cpp_doxygen = false # true/false - -# If true, space is added with sp_cmt_cpp_start will be added after Qt -# translator or meta-data comments like '//:', '//=', and '//~'. -sp_cmt_cpp_qttr = false # true/false - -# Add or remove space between #else or #endif and a trailing comment. -sp_endif_cmt = ignore # ignore/add/remove/force - -# Add or remove space after 'new', 'delete' and 'delete[]'. -sp_after_new = ignore # ignore/add/remove/force - -# Add or remove space between 'new' and '(' in 'new()'. -sp_between_new_paren = ignore # ignore/add/remove/force - -# Add or remove space between ')' and type in 'new(foo) BAR'. -sp_after_newop_paren = ignore # ignore/add/remove/force - -# Add or remove space inside parenthesis of the new operator -# as in 'new(foo) BAR'. -sp_inside_newop_paren = ignore # ignore/add/remove/force - -# Add or remove space after the open parenthesis of the new operator, -# as in 'new(foo) BAR'. -# -# Overrides sp_inside_newop_paren. -sp_inside_newop_paren_open = ignore # ignore/add/remove/force - -# Add or remove space before the close parenthesis of the new operator, -# as in 'new(foo) BAR'. -# -# Overrides sp_inside_newop_paren. -sp_inside_newop_paren_close = ignore # ignore/add/remove/force - -# Add or remove space before a trailing or embedded comment. -sp_before_tr_emb_cmt = ignore # ignore/add/remove/force - -# Number of spaces before a trailing or embedded comment. -sp_num_before_tr_emb_cmt = 0 # unsigned number - -# (Java) Add or remove space between an annotation and the open parenthesis. -sp_annotation_paren = ignore # ignore/add/remove/force - -# If true, vbrace tokens are dropped to the previous token and skipped. -sp_skip_vbrace_tokens = false # true/false - -# Add or remove space after 'noexcept'. -sp_after_noexcept = ignore # ignore/add/remove/force - -# Add or remove space after '_'. -sp_vala_after_translation = ignore # ignore/add/remove/force - -# If true, a is inserted after #define. -force_tab_after_define = false # true/false - -# -# Indenting options -# - -# The number of columns to indent per level. Usually 2, 3, 4, or 8. -# -# Default: 8 -indent_columns = 4 # unsigned number - -# The continuation indent. If non-zero, this overrides the indent of '(', '[' -# and '=' continuation indents. Negative values are OK; negative value is -# absolute and not increased for each '(' or '[' level. -# -# For FreeBSD, this is set to 4. -indent_continue = 0 # number - -# The continuation indent, only for class header line(s). If non-zero, this -# overrides the indent of 'class' continuation indents. -indent_continue_class_head = 0 # unsigned number - -# Whether to indent empty lines (i.e. lines which contain only spaces before -# the newline character). -indent_single_newlines = false # true/false - -# The continuation indent for func_*_param if they are true. If non-zero, this -# overrides the indent. -indent_param = 0 # unsigned number - -# How to use tabs when indenting code. -# -# 0: Spaces only -# 1: Indent with tabs to brace level, align with spaces (default) -# 2: Indent and align with tabs, using spaces when not on a tabstop -# -# Default: 1 -indent_with_tabs = 0 # unsigned number - -# Whether to indent comments that are not at a brace level with tabs on a -# tabstop. Requires indent_with_tabs=2. If false, will use spaces. -indent_cmt_with_tabs = false # true/false - -# Whether to indent strings broken by '\' so that they line up. -indent_align_string = false # true/false - -# The number of spaces to indent multi-line XML strings. -# Requires indent_align_string=true. -indent_xml_string = 0 # unsigned number - -# Spaces to indent '{' from level. -indent_brace = 0 # unsigned number - -# Whether braces are indented to the body level. -indent_braces = false # true/false - -# Whether to disable indenting function braces if indent_braces=true. -indent_braces_no_func = false # true/false - -# Whether to disable indenting class braces if indent_braces=true. -indent_braces_no_class = false # true/false - -# Whether to disable indenting struct braces if indent_braces=true. -indent_braces_no_struct = false # true/false - -# Whether to indent based on the size of the brace parent, -# i.e. 'if' => 3 spaces, 'for' => 4 spaces, etc. -indent_brace_parent = false # true/false - -# Whether to indent based on the open parenthesis instead of the open brace -# in '({\n'. -indent_paren_open_brace = false # true/false - -# (C#) Whether to indent the brace of a C# delegate by another level. -indent_cs_delegate_brace = false # true/false - -# (C#) Whether to indent a C# delegate (to handle delegates with no brace) by -# another level. -indent_cs_delegate_body = false # true/false - -# Whether to indent the body of a 'namespace'. -indent_namespace = false # true/false - -# Whether to indent only the first namespace, and not any nested namespaces. -# Requires indent_namespace=true. -indent_namespace_single_indent = false # true/false - -# The number of spaces to indent a namespace block. -# If set to zero, use the value indent_columns -indent_namespace_level = 0 # unsigned number - -# If the body of the namespace is longer than this number, it won't be -# indented. Requires indent_namespace=true. 0 means no limit. -indent_namespace_limit = 0 # unsigned number - -# Whether the 'extern "C"' body is indented. -indent_extern = false # true/false - -# Whether the 'class' body is indented. -indent_class = true # true/false - -# Whether to indent the stuff after a leading base class colon. -indent_class_colon = false # true/false - -# Whether to indent based on a class colon instead of the stuff after the -# colon. Requires indent_class_colon=true. -indent_class_on_colon = false # true/false - -# Whether to indent the stuff after a leading class initializer colon. -indent_constr_colon = false # true/false - -# Virtual indent from the ':' for member initializers. -# -# Default: 2 -indent_ctor_init_leading = 2 # unsigned number - -# Additional indent for constructor initializer list. -# Negative values decrease indent down to the first column. -indent_ctor_init = 0 # number - -# Whether to indent 'if' following 'else' as a new block under the 'else'. -# If false, 'else\nif' is treated as 'else if' for indenting purposes. -indent_else_if = false # true/false - -# Amount to indent variable declarations after a open brace. -# -# <0: Relative -# >=0: Absolute -indent_var_def_blk = 0 # number - -# Whether to indent continued variable declarations instead of aligning. -indent_var_def_cont = true # true/false - -# Whether to indent continued shift expressions ('<<' and '>>') instead of -# aligning. Set align_left_shift=false when enabling this. -indent_shift = false # true/false - -# Whether to force indentation of function definitions to start in column 1. -indent_func_def_force_col1 = true # true/false - -# Whether to indent continued function call parameters one indent level, -# rather than aligning parameters under the open parenthesis. -indent_func_call_param = true # true/false - -# Whether to indent continued function definition parameters one indent level, -# rather than aligning parameters under the open parenthesis. -indent_func_def_param = true # true/false - -# for function definitions, only if indent_func_def_param is false -# Allows to align params when appropriate and indent them when not -# behave as if it was true if paren position is more than this value -# if paren position is more than the option value -indent_func_def_param_paren_pos_threshold = 0 # unsigned number - -# Whether to indent continued function call prototype one indent level, -# rather than aligning parameters under the open parenthesis. -indent_func_proto_param = true # true/false - -# Whether to indent continued function call declaration one indent level, -# rather than aligning parameters under the open parenthesis. -indent_func_class_param = true # true/false - -# Whether to indent continued class variable constructors one indent level, -# rather than aligning parameters under the open parenthesis. -indent_func_ctor_var_param = true # true/false - -# Whether to indent continued template parameter list one indent level, -# rather than aligning parameters under the open parenthesis. -indent_template_param = true # true/false - -# Double the indent for indent_func_xxx_param options. -# Use both values of the options indent_columns and indent_param. -indent_func_param_double = false # true/false - -# Indentation column for standalone 'const' qualifier on a function -# prototype. -indent_func_const = 0 # unsigned number - -# Indentation column for standalone 'throw' qualifier on a function -# prototype. -indent_func_throw = 0 # unsigned number - -# How to indent within a macro followed by a brace on the same line -# This allows reducing the indent in macros that have (for example) -# `do { ... } while (0)` blocks bracketing them. -# -# true: add an indent for the brace on the same line as the macro -# false: do not add an indent for the brace on the same line as the macro -# -# Default: true -indent_macro_brace = true # true/false - -# The number of spaces to indent a continued '->' or '.'. -# Usually set to 0, 1, or indent_columns. -indent_member = 0 # unsigned number - -# Whether lines broken at '.' or '->' should be indented by a single indent. -# The indent_member option will not be effective if this is set to true. -indent_member_single = false # true/false - -# Spaces to indent single line ('//') comments on lines before code. -indent_sing_line_comments = 0 # unsigned number - -# Whether to indent trailing single line ('//') comments relative to the code -# instead of trying to keep the same absolute column. -indent_relative_single_line_comments = false # true/false - -# Spaces to indent 'case' from 'switch'. Usually 0 or indent_columns. -indent_switch_case = indent_columns # unsigned number - -# indent 'break' with 'case' from 'switch'. -indent_switch_break_with_case = false # true/false - -# Whether to indent preprocessor statements inside of switch statements. -# -# Default: true -indent_switch_pp = true # true/false - -# Spaces to shift the 'case' line, without affecting any other lines. -# Usually 0. -indent_case_shift = 0 # unsigned number - -# Spaces to indent '{' from 'case'. By default, the brace will appear under -# the 'c' in case. Usually set to 0 or indent_columns. Negative values are OK. -indent_case_brace = 0 # number - -# Whether to indent comments found in first column. -indent_col1_comment = false # true/false - -# Whether to indent multi string literal in first column. -indent_col1_multi_string_literal = false # true/false - -# How to indent goto labels. -# -# >0: Absolute column where 1 is the leftmost column -# <=0: Subtract from brace indent -# -# Default: 1 -indent_label = 1 # number - -# How to indent access specifiers that are followed by a -# colon. -# -# >0: Absolute column where 1 is the leftmost column -# <=0: Subtract from brace indent -# -# Default: 1 -indent_access_spec = -4 # number - -# Whether to indent the code after an access specifier by one level. -# If true, this option forces 'indent_access_spec=0'. -indent_access_spec_body = false # true/false - -# If an open parenthesis is followed by a newline, whether to indent the next -# line so that it lines up after the open parenthesis (not recommended). -indent_paren_nl = false # true/false - -# How to indent a close parenthesis after a newline. -# -# 0: Indent to body level (default) -# 1: Align under the open parenthesis -# 2: Indent to the brace level -indent_paren_close = 0 # unsigned number - -# Whether to indent the open parenthesis of a function definition, -# if the parenthesis is on its own line. -indent_paren_after_func_def = false # true/false - -# Whether to indent the open parenthesis of a function declaration, -# if the parenthesis is on its own line. -indent_paren_after_func_decl = false # true/false - -# Whether to indent the open parenthesis of a function call, -# if the parenthesis is on its own line. -indent_paren_after_func_call = false # true/false - -# Whether to indent a comma when inside a parenthesis. -# If true, aligns under the open parenthesis. -indent_comma_paren = false # true/false - -# Whether to indent a Boolean operator when inside a parenthesis. -# If true, aligns under the open parenthesis. -indent_bool_paren = false # true/false - -# Whether to indent a semicolon when inside a for parenthesis. -# If true, aligns under the open for parenthesis. -indent_semicolon_for_paren = false # true/false - -# Whether to align the first expression to following ones -# if indent_bool_paren=true. -indent_first_bool_expr = false # true/false - -# Whether to align the first expression to following ones -# if indent_semicolon_for_paren=true. -indent_first_for_expr = false # true/false - -# If an open square is followed by a newline, whether to indent the next line -# so that it lines up after the open square (not recommended). -indent_square_nl = false # true/false - -# (ESQL/C) Whether to preserve the relative indent of 'EXEC SQL' bodies. -indent_preserve_sql = false # true/false - -# Whether to align continued statements at the '='. If false or if the '=' is -# followed by a newline, the next line is indent one tab. -# -# Default: true -indent_align_assign = false # true/false - -# Whether to align continued statements at the '('. If false or the '(' is -# followed by a newline, the next line indent is one tab. -# -# Default: true -indent_align_paren = true # true/false, - -# (OC) Whether to indent Objective-C blocks at brace level instead of usual -# rules. -indent_oc_block = false # true/false - -# (OC) Indent for Objective-C blocks in a message relative to the parameter -# name. -# -# =0: Use indent_oc_block rules -# >0: Use specified number of spaces to indent -indent_oc_block_msg = 0 # unsigned number - -# (OC) Minimum indent for subsequent parameters -indent_oc_msg_colon = 0 # unsigned number - -# (OC) Whether to prioritize aligning with initial colon (and stripping spaces -# from lines, if necessary). -# -# Default: true -indent_oc_msg_prioritize_first_colon = true # true/false - -# (OC) Whether to indent blocks the way that Xcode does by default -# (from the keyword if the parameter is on its own line; otherwise, from the -# previous indentation level). Requires indent_oc_block_msg=true. -indent_oc_block_msg_xcode_style = false # true/false - -# (OC) Whether to indent blocks from where the brace is, relative to a -# message keyword. Requires indent_oc_block_msg=true. -indent_oc_block_msg_from_keyword = false # true/false - -# (OC) Whether to indent blocks from where the brace is, relative to a message -# colon. Requires indent_oc_block_msg=true. -indent_oc_block_msg_from_colon = false # true/false - -# (OC) Whether to indent blocks from where the block caret is. -# Requires indent_oc_block_msg=true. -indent_oc_block_msg_from_caret = false # true/false - -# (OC) Whether to indent blocks from where the brace caret is. -# Requires indent_oc_block_msg=true. -indent_oc_block_msg_from_brace = false # true/false - -# When indenting after virtual brace open and newline add further spaces to -# reach this minimum indent. -indent_min_vbrace_open = 0 # unsigned number - -# Whether to add further spaces after regular indent to reach next tabstop -# when identing after virtual brace open and newline. -indent_vbrace_open_on_tabstop = false # true/false - -# How to indent after a brace followed by another token (not a newline). -# true: indent all contained lines to match the token -# false: indent all contained lines to match the brace -# -# Default: true -indent_token_after_brace = true # true/false - -# Whether to indent the body of a C++11 lambda. -indent_cpp_lambda_body = false # true/false - -# (C#) Whether to indent a 'using' block if no braces are used. -# -# Default: true -indent_using_block = true # true/false - -# How to indent the continuation of ternary operator. -# -# 0: Off (default) -# 1: When the `if_false` is a continuation, indent it under `if_false` -# 2: When the `:` is a continuation, indent it under `?` -indent_ternary_operator = 0 # unsigned number - -# If true, the indentation of the chunks after a `return new` sequence will be set at return indentation column. -indent_off_after_return_new = false # true/false - -# If true, the tokens after return are indented with regular single indentation. -# By default (false) the indentation is after the return token. -indent_single_after_return = false # true/false - -# Whether to ignore indent and alignment for 'asm' blocks (i.e. assume they -# have their own indentation). -indent_ignore_asm_block = false # true/false - -# -# Newline adding and removing options -# - -# Whether to collapse empty blocks between '{' and '}'. -nl_collapse_empty_body = false # true/false - -# Don't split one-line braced assignments, as in 'foo_t f = { 1, 2 };'. -nl_assign_leave_one_liners = true # true/false - -# Don't split one-line braced statements inside a 'class xx { }' body. -nl_class_leave_one_liners = true # true/false - -# Don't split one-line enums, as in 'enum foo { BAR = 15 };' -nl_enum_leave_one_liners = true # true/false - -# Don't split one-line get or set functions. -nl_getset_leave_one_liners = true # true/false - -# (C#) Don't split one-line property get or set functions. -nl_cs_property_leave_one_liners = true # true/false - -# Don't split one-line function definitions, as in 'int foo() { return 0; }'. -# night modify nl_func_type_name -nl_func_leave_one_liners = true # true/false - -# Don't split one-line C++11 lambdas, as in '[]() { return 0; }'. -nl_cpp_lambda_leave_one_liners = true # true/false - -# Don't split one-line if/else statements, as in 'if(...) b++;'. -nl_if_leave_one_liners = true # true/false - -# Don't split one-line while statements, as in 'while(...) b++;'. -nl_while_leave_one_liners = true # true/false - -# Don't split one-line for statements, as in 'for(...) b++;'. -nl_for_leave_one_liners = true # true/false - -# (OC) Don't split one-line Objective-C messages. -nl_oc_msg_leave_one_liner = false # true/false - -# (OC) Add or remove newline between method declaration and '{'. -nl_oc_mdef_brace = ignore # ignore/add/remove/force - -# (OC) Add or remove newline between Objective-C block signature and '{'. -nl_oc_block_brace = ignore # ignore/add/remove/force - -# (OC) Add or remove blank line before '@interface' statement. -nl_oc_before_interface = ignore # ignore/add/remove/force - -# (OC) Add or remove blank line before '@implementation' statement. -nl_oc_before_implementation = ignore # ignore/add/remove/force - -# (OC) Add or remove blank line before '@end' statement. -nl_oc_before_end = ignore # ignore/add/remove/force - -# (OC) Add or remove newline between '@interface' and '{'. -nl_oc_interface_brace = ignore # ignore/add/remove/force - -# (OC) Add or remove newline between '@implementation' and '{'. -nl_oc_implementation_brace = ignore # ignore/add/remove/force - -# Add or remove newlines at the start of the file. -nl_start_of_file = ignore # ignore/add/remove/force - -# The minimum number of newlines at the start of the file (only used if -# nl_start_of_file is 'add' or 'force'). -nl_start_of_file_min = 0 # unsigned number - -# Add or remove newline at the end of the file. -nl_end_of_file = ignore # ignore/add/remove/force - -# The minimum number of newlines at the end of the file (only used if -# nl_end_of_file is 'add' or 'force'). -nl_end_of_file_min = 0 # unsigned number - -# Add or remove newline between '=' and '{'. -nl_assign_brace = ignore # ignore/add/remove/force - -# (D) Add or remove newline between '=' and '['. -nl_assign_square = ignore # ignore/add/remove/force - -# Add or remove newline between '[]' and '{'. -nl_tsquare_brace = ignore # ignore/add/remove/force - -# (D) Add or remove newline after '= ['. Will also affect the newline before -# the ']'. -nl_after_square_assign = ignore # ignore/add/remove/force - -# Add or remove newline between a function call's ')' and '{', as in -# 'list_for_each(item, &list) { }'. -nl_fcall_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'enum' and '{'. -nl_enum_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'enum' and 'class'. -nl_enum_class = ignore # ignore/add/remove/force - -# Add or remove newline between 'enum class' and the identifier. -nl_enum_class_identifier = ignore # ignore/add/remove/force - -# Add or remove newline between 'enum class' type and ':'. -nl_enum_identifier_colon = ignore # ignore/add/remove/force - -# Add or remove newline between 'enum class identifier :' and type. -nl_enum_colon_type = ignore # ignore/add/remove/force - -# Add or remove newline between 'struct and '{'. -nl_struct_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'union' and '{'. -nl_union_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'if' and '{'. -nl_if_brace = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and 'else'. -nl_brace_else = ignore # ignore/add/remove/force - -# Add or remove newline between 'else if' and '{'. If set to ignore, -# nl_if_brace is used instead. -nl_elseif_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'else' and '{'. -nl_else_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'else' and 'if'. -nl_else_if = ignore # ignore/add/remove/force - -# Add or remove newline before 'if'/'else if' closing parenthesis. -nl_before_if_closing_paren = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and 'finally'. -nl_brace_finally = ignore # ignore/add/remove/force - -# Add or remove newline between 'finally' and '{'. -nl_finally_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'try' and '{'. -nl_try_brace = ignore # ignore/add/remove/force - -# Add or remove newline between get/set and '{'. -nl_getset_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'for' and '{'. -nl_for_brace = ignore # ignore/add/remove/force - -# Add or remove newline before the '{' of a 'catch' statement, as in -# 'catch (decl) {'. -nl_catch_brace = ignore # ignore/add/remove/force - -# (OC) Add or remove newline before the '{' of a '@catch' statement, as in -# '@catch (decl) {'. If set to ignore, nl_catch_brace is used. -nl_oc_catch_brace = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and 'catch'. -nl_brace_catch = ignore # ignore/add/remove/force - -# (OC) Add or remove newline between '}' and '@catch'. If set to ignore, -# nl_brace_catch is used. -nl_oc_brace_catch = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and ']'. -nl_brace_square = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and ')' in a function invocation. -nl_brace_fparen = ignore # ignore/add/remove/force - -# Add or remove newline between 'while' and '{'. -nl_while_brace = ignore # ignore/add/remove/force - -# (D) Add or remove newline between 'scope (x)' and '{'. -nl_scope_brace = ignore # ignore/add/remove/force - -# (D) Add or remove newline between 'unittest' and '{'. -nl_unittest_brace = ignore # ignore/add/remove/force - -# (D) Add or remove newline between 'version (x)' and '{'. -nl_version_brace = ignore # ignore/add/remove/force - -# (C#) Add or remove newline between 'using' and '{'. -nl_using_brace = ignore # ignore/add/remove/force - -# Add or remove newline between two open or close braces. Due to general -# newline/brace handling, REMOVE may not work. -nl_brace_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'do' and '{'. -nl_do_brace = ignore # ignore/add/remove/force - -# Add or remove newline between '}' and 'while' of 'do' statement. -nl_brace_while = ignore # ignore/add/remove/force - -# Add or remove newline between 'switch' and '{'. -nl_switch_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'synchronized' and '{'. -nl_synchronized_brace = ignore # ignore/add/remove/force - -# Add a newline between ')' and '{' if the ')' is on a different line than the -# if/for/etc. -# -# Overrides nl_for_brace, nl_if_brace, nl_switch_brace, nl_while_switch and -# nl_catch_brace. -nl_multi_line_cond = false # true/false - -# Add a newline after '(' if an if/for/while/switch condition spans multiple -# lines -nl_multi_line_sparen_open = ignore # ignore/add/remove/force - -# Add a newline before ')' if an if/for/while/switch condition spans multiple -# lines. Overrides nl_before_if_closing_paren if both are specified. -nl_multi_line_sparen_close = ignore # ignore/add/remove/force - -# Force a newline in a define after the macro name for multi-line defines. -nl_multi_line_define = false # true/false - -# Whether to add a newline before 'case', and a blank line before a 'case' -# statement that follows a ';' or '}'. -nl_before_case = false # true/false - -# Whether to add a newline after a 'case' statement. -nl_after_case = false # true/false - -# Add or remove newline between a case ':' and '{'. -# -# Overrides nl_after_case. -nl_case_colon_brace = ignore # ignore/add/remove/force - -# Add or remove newline between ')' and 'throw'. -nl_before_throw = ignore # ignore/add/remove/force - -# Add or remove newline between 'namespace' and '{'. -nl_namespace_brace = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template class. -nl_template_class = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template class declaration. -# -# Overrides nl_template_class. -nl_template_class_decl = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<>' of a specialized class declaration. -# -# Overrides nl_template_class_decl. -nl_template_class_decl_special = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template class definition. -# -# Overrides nl_template_class. -nl_template_class_def = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<>' of a specialized class definition. -# -# Overrides nl_template_class_def. -nl_template_class_def_special = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template function. -nl_template_func = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template function -# declaration. -# -# Overrides nl_template_func. -nl_template_func_decl = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<>' of a specialized function -# declaration. -# -# Overrides nl_template_func_decl. -nl_template_func_decl_special = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template function -# definition. -# -# Overrides nl_template_func. -nl_template_func_def = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<>' of a specialized function -# definition. -# -# Overrides nl_template_func_def. -nl_template_func_def_special = ignore # ignore/add/remove/force - -# Add or remove newline after 'template<...>' of a template variable. -nl_template_var = ignore # ignore/add/remove/force - -# Add or remove newline between 'template<...>' and 'using' of a templated -# type alias. -nl_template_using = ignore # ignore/add/remove/force - -# Add or remove newline between 'class' and '{'. -nl_class_brace = ignore # ignore/add/remove/force - -# Add or remove newline before or after (depending on pos_class_comma, -# may not be IGNORE) each',' in the base class list. -nl_class_init_args = ignore # ignore/add/remove/force - -# Add or remove newline after each ',' in the constructor member -# initialization. Related to nl_constr_colon, pos_constr_colon and -# pos_constr_comma. -nl_constr_init_args = ignore # ignore/add/remove/force - -# Add or remove newline before first element, after comma, and after last -# element, in 'enum'. -nl_enum_own_lines = ignore # ignore/add/remove/force - -# Add or remove newline between return type and function name in a function -# definition. -# might be modified by nl_func_leave_one_liners -nl_func_type_name = ignore # ignore/add/remove/force - -# Add or remove newline between return type and function name inside a class -# definition. If set to ignore, nl_func_type_name or nl_func_proto_type_name -# is used instead. -nl_func_type_name_class = ignore # ignore/add/remove/force - -# Add or remove newline between class specification and '::' -# in 'void A::f() { }'. Only appears in separate member implementation (does -# not appear with in-line implementation). -nl_func_class_scope = ignore # ignore/add/remove/force - -# Add or remove newline between function scope and name, as in -# 'void A :: f() { }'. -nl_func_scope_name = ignore # ignore/add/remove/force - -# Add or remove newline between return type and function name in a prototype. -nl_func_proto_type_name = ignore # ignore/add/remove/force - -# Add or remove newline between a function name and the opening '(' in the -# declaration. -nl_func_paren = ignore # ignore/add/remove/force - -# Overrides nl_func_paren for functions with no parameters. -nl_func_paren_empty = ignore # ignore/add/remove/force - -# Add or remove newline between a function name and the opening '(' in the -# definition. -nl_func_def_paren = ignore # ignore/add/remove/force - -# Overrides nl_func_def_paren for functions with no parameters. -nl_func_def_paren_empty = ignore # ignore/add/remove/force - -# Add or remove newline between a function name and the opening '(' in the -# call. -nl_func_call_paren = ignore # ignore/add/remove/force - -# Overrides nl_func_call_paren for functions with no parameters. -nl_func_call_paren_empty = ignore # ignore/add/remove/force - -# Add or remove newline after '(' in a function declaration. -nl_func_decl_start = ignore # ignore/add/remove/force - -# Add or remove newline after '(' in a function definition. -nl_func_def_start = ignore # ignore/add/remove/force - -# Overrides nl_func_decl_start when there is only one parameter. -nl_func_decl_start_single = ignore # ignore/add/remove/force - -# Overrides nl_func_def_start when there is only one parameter. -nl_func_def_start_single = ignore # ignore/add/remove/force - -# Whether to add a newline after '(' in a function declaration if '(' and ')' -# are in different lines. If false, nl_func_decl_start is used instead. -nl_func_decl_start_multi_line = false # true/false - -# Whether to add a newline after '(' in a function definition if '(' and ')' -# are in different lines. If false, nl_func_def_start is used instead. -nl_func_def_start_multi_line = false # true/false - -# Add or remove newline after each ',' in a function declaration. -nl_func_decl_args = ignore # ignore/add/remove/force - -# Add or remove newline after each ',' in a function definition. -nl_func_def_args = ignore # ignore/add/remove/force - -# Whether to add a newline after each ',' in a function declaration if '(' -# and ')' are in different lines. If false, nl_func_decl_args is used instead. -nl_func_decl_args_multi_line = false # true/false - -# Whether to add a newline after each ',' in a function definition if '(' -# and ')' are in different lines. If false, nl_func_def_args is used instead. -nl_func_def_args_multi_line = false # true/false - -# Add or remove newline before the ')' in a function declaration. -nl_func_decl_end = ignore # ignore/add/remove/force - -# Add or remove newline before the ')' in a function definition. -nl_func_def_end = ignore # ignore/add/remove/force - -# Overrides nl_func_decl_end when there is only one parameter. -nl_func_decl_end_single = ignore # ignore/add/remove/force - -# Overrides nl_func_def_end when there is only one parameter. -nl_func_def_end_single = ignore # ignore/add/remove/force - -# Whether to add a newline before ')' in a function declaration if '(' and ')' -# are in different lines. If false, nl_func_decl_end is used instead. -nl_func_decl_end_multi_line = false # true/false - -# Whether to add a newline before ')' in a function definition if '(' and ')' -# are in different lines. If false, nl_func_def_end is used instead. -nl_func_def_end_multi_line = false # true/false - -# Add or remove newline between '()' in a function declaration. -nl_func_decl_empty = ignore # ignore/add/remove/force - -# Add or remove newline between '()' in a function definition. -nl_func_def_empty = ignore # ignore/add/remove/force - -# Add or remove newline between '()' in a function call. -nl_func_call_empty = ignore # ignore/add/remove/force - -# Whether to add a newline after '(' in a function call, -# has preference over nl_func_call_start_multi_line. -nl_func_call_start = ignore # ignore/add/remove/force - -# Whether to add a newline after '(' in a function call if '(' and ')' are in -# different lines. -nl_func_call_start_multi_line = false # true/false - -# Whether to add a newline after each ',' in a function call if '(' and ')' -# are in different lines. -nl_func_call_args_multi_line = false # true/false - -# Whether to add a newline before ')' in a function call if '(' and ')' are in -# different lines. -nl_func_call_end_multi_line = false # true/false - -# Whether to add a newline after '<' of a template parameter list. -nl_template_start = false # true/false - -# Whether to add a newline after each ',' in a template parameter list. -nl_template_args = false # true/false - -# Whether to add a newline before '>' of a template parameter list. -nl_template_end = false # true/false - -# (OC) Whether to put each Objective-C message parameter on a separate line. -# See nl_oc_msg_leave_one_liner. -nl_oc_msg_args = false # true/false - -# Add or remove newline between function signature and '{'. -nl_fdef_brace = ignore # ignore/add/remove/force - -# Add or remove newline between function signature and '{', -# if signature ends with ')'. Overrides nl_fdef_brace. -nl_fdef_brace_cond = ignore # ignore/add/remove/force - -# Add or remove newline between C++11 lambda signature and '{'. -nl_cpp_ldef_brace = ignore # ignore/add/remove/force - -# Add or remove newline between 'return' and the return expression. -nl_return_expr = ignore # ignore/add/remove/force - -# Whether to add a newline after semicolons, except in 'for' statements. -nl_after_semicolon = false # true/false - -# (Java) Add or remove newline between the ')' and '{{' of the double brace -# initializer. -nl_paren_dbrace_open = ignore # ignore/add/remove/force - -# Whether to add a newline after the type in an unnamed temporary -# direct-list-initialization. -nl_type_brace_init_lst = ignore # ignore/add/remove/force - -# Whether to add a newline after the open brace in an unnamed temporary -# direct-list-initialization. -nl_type_brace_init_lst_open = ignore # ignore/add/remove/force - -# Whether to add a newline before the close brace in an unnamed temporary -# direct-list-initialization. -nl_type_brace_init_lst_close = ignore # ignore/add/remove/force - -# Whether to add a newline after '{'. This also adds a newline before the -# matching '}'. -nl_after_brace_open = false # true/false - -# Whether to add a newline between the open brace and a trailing single-line -# comment. Requires nl_after_brace_open=true. -nl_after_brace_open_cmt = false # true/false - -# Whether to add a newline after a virtual brace open with a non-empty body. -# These occur in un-braced if/while/do/for statement bodies. -nl_after_vbrace_open = false # true/false - -# Whether to add a newline after a virtual brace open with an empty body. -# These occur in un-braced if/while/do/for statement bodies. -nl_after_vbrace_open_empty = false # true/false - -# Whether to add a newline after '}'. Does not apply if followed by a -# necessary ';'. -nl_after_brace_close = false # true/false - -# Whether to add a newline after a virtual brace close, -# as in 'if (foo) a++; return;'. -nl_after_vbrace_close = false # true/false - -# Add or remove newline between the close brace and identifier, -# as in 'struct { int a; } b;'. Affects enumerations, unions and -# structures. If set to ignore, uses nl_after_brace_close. -nl_brace_struct_var = ignore # ignore/add/remove/force - -# Whether to alter newlines in '#define' macros. -nl_define_macro = false # true/false - -# Whether to alter newlines between consecutive parenthesis closes. The number -# of closing parentheses in a line will depend on respective open parenthesis -# lines. -nl_squeeze_paren_close = false # true/false - -# Whether to remove blanks after '#ifxx' and '#elxx', or before '#elxx' and -# '#endif'. Does not affect top-level #ifdefs. -nl_squeeze_ifdef = false # true/false - -# Makes the nl_squeeze_ifdef option affect the top-level #ifdefs as well. -nl_squeeze_ifdef_top_level = false # true/false - -# Add or remove blank line before 'if'. -nl_before_if = ignore # ignore/add/remove/force - -# Add or remove blank line after 'if' statement. Add/Force work only if the -# next token is not a closing brace. -nl_after_if = ignore # ignore/add/remove/force - -# Add or remove blank line before 'for'. -nl_before_for = ignore # ignore/add/remove/force - -# Add or remove blank line after 'for' statement. -nl_after_for = ignore # ignore/add/remove/force - -# Add or remove blank line before 'while'. -nl_before_while = ignore # ignore/add/remove/force - -# Add or remove blank line after 'while' statement. -nl_after_while = ignore # ignore/add/remove/force - -# Add or remove blank line before 'switch'. -nl_before_switch = ignore # ignore/add/remove/force - -# Add or remove blank line after 'switch' statement. -nl_after_switch = ignore # ignore/add/remove/force - -# Add or remove blank line before 'synchronized'. -nl_before_synchronized = ignore # ignore/add/remove/force - -# Add or remove blank line after 'synchronized' statement. -nl_after_synchronized = ignore # ignore/add/remove/force - -# Add or remove blank line before 'do'. -nl_before_do = ignore # ignore/add/remove/force - -# Add or remove blank line after 'do/while' statement. -nl_after_do = ignore # ignore/add/remove/force - -# Whether to put a blank line before 'return' statements, unless after an open -# brace. -nl_before_return = false # true/false - -# Whether to put a blank line after 'return' statements, unless followed by a -# close brace. -nl_after_return = false # true/false - -# (Java) Whether to put a blank line before a member '.' or '->' operators. -nl_before_member = ignore # ignore/add/remove/force - -# (Java) Whether to put a blank line after a member '.' or '->' operators. -nl_after_member = ignore # ignore/add/remove/force - -# Whether to double-space commented-entries in 'struct'/'union'/'enum'. -nl_ds_struct_enum_cmt = false # true/false - -# Whether to force a newline before '}' of a 'struct'/'union'/'enum'. -# (Lower priority than eat_blanks_before_close_brace.) -nl_ds_struct_enum_close_brace = false # true/false - -# Add or remove newline before or after (depending on pos_class_colon) a class -# colon, as in 'class Foo : public Bar'. -nl_class_colon = ignore # ignore/add/remove/force - -# Add or remove newline around a class constructor colon. The exact position -# depends on nl_constr_init_args, pos_constr_colon and pos_constr_comma. -nl_constr_colon = ignore # ignore/add/remove/force - -# Whether to collapse a two-line namespace, like 'namespace foo\n{ decl; }' -# into a single line. If true, prevents other brace newline rules from turning -# such code into four lines. -nl_namespace_two_to_one_liner = false # true/false - -# Whether to remove a newline in simple unbraced if statements, turning them -# into one-liners, as in 'if(b)\n i++;' => 'if(b) i++;'. -nl_create_if_one_liner = false # true/false - -# Whether to remove a newline in simple unbraced for statements, turning them -# into one-liners, as in 'for (...)\n stmt;' => 'for (...) stmt;'. -nl_create_for_one_liner = false # true/false - -# Whether to remove a newline in simple unbraced while statements, turning -# them into one-liners, as in 'while (expr)\n stmt;' => 'while (expr) stmt;'. -nl_create_while_one_liner = false # true/false - -# Whether to collapse a function definition whose body (not counting braces) -# is only one line so that the entire definition (prototype, braces, body) is -# a single line. -nl_create_func_def_one_liner = false # true/false - -# Whether to collapse a function definition whose body (not counting braces) -# is only one line so that the entire definition (prototype, braces, body) is -# a single line. -nl_create_list_one_liner = false # true/false - -# Whether to split one-line simple unbraced if statements into two lines by -# adding a newline, as in 'if(b) i++;'. -nl_split_if_one_liner = false # true/false - -# Whether to split one-line simple unbraced for statements into two lines by -# adding a newline, as in 'for (...) stmt;'. -nl_split_for_one_liner = false # true/false - -# Whether to split one-line simple unbraced while statements into two lines by -# adding a newline, as in 'while (expr) stmt;'. -nl_split_while_one_liner = false # true/false - -# -# Blank line options -# - -# The maximum number of consecutive newlines (3 = 2 blank lines). -nl_max = 0 # unsigned number - -# The maximum number of consecutive newlines in a function. -nl_max_blank_in_func = 0 # unsigned number - -# The number of newlines before a function prototype. -nl_before_func_body_proto = 0 # unsigned number - -# The number of newlines before a multi-line function definition. -nl_before_func_body_def = 0 # unsigned number - -# The number of newlines before a class constructor/destructor prototype. -nl_before_func_class_proto = 0 # unsigned number - -# The number of newlines before a class constructor/destructor definition. -nl_before_func_class_def = 0 # unsigned number - -# The number of newlines after a function prototype. -nl_after_func_proto = 0 # unsigned number - -# The number of newlines after a function prototype, if not followed by -# another function prototype. -nl_after_func_proto_group = 0 # unsigned number - -# The number of newlines after a class constructor/destructor prototype. -nl_after_func_class_proto = 0 # unsigned number - -# The number of newlines after a class constructor/destructor prototype, -# if not followed by another constructor/destructor prototype. -nl_after_func_class_proto_group = 0 # unsigned number - -# Whether one-line method definitions inside a class body should be treated -# as if they were prototypes for the purposes of adding newlines. -# -# Requires nl_class_leave_one_liners=true. Overrides nl_before_func_body_def -# and nl_before_func_class_def for one-liners. -nl_class_leave_one_liner_groups = false # true/false - -# The number of newlines after '}' of a multi-line function body. -nl_after_func_body = 0 # unsigned number - -# The number of newlines after '}' of a multi-line function body in a class -# declaration. Also affects class constructors/destructors. -# -# Overrides nl_after_func_body. -nl_after_func_body_class = 0 # unsigned number - -# The number of newlines after '}' of a single line function body. Also -# affects class constructors/destructors. -# -# Overrides nl_after_func_body and nl_after_func_body_class. -nl_after_func_body_one_liner = 0 # unsigned number - -# The number of blank lines after a block of variable definitions at the top -# of a function body. -# -# 0: No change (default). -nl_func_var_def_blk = 0 # unsigned number - -# The number of newlines before a block of typedefs. If nl_after_access_spec -# is non-zero, that option takes precedence. -# -# 0: No change (default). -nl_typedef_blk_start = 0 # unsigned number - -# The number of newlines after a block of typedefs. -# -# 0: No change (default). -nl_typedef_blk_end = 0 # unsigned number - -# The maximum number of consecutive newlines within a block of typedefs. -# -# 0: No change (default). -nl_typedef_blk_in = 0 # unsigned number - -# The number of newlines before a block of variable definitions not at the top -# of a function body. If nl_after_access_spec is non-zero, that option takes -# precedence. -# -# 0: No change (default). -nl_var_def_blk_start = 0 # unsigned number - -# The number of newlines after a block of variable definitions not at the top -# of a function body. -# -# 0: No change (default). -nl_var_def_blk_end = 0 # unsigned number - -# The maximum number of consecutive newlines within a block of variable -# definitions. -# -# 0: No change (default). -nl_var_def_blk_in = 0 # unsigned number - -# The minimum number of newlines before a multi-line comment. -# Doesn't apply if after a brace open or another multi-line comment. -nl_before_block_comment = 0 # unsigned number - -# The minimum number of newlines before a single-line C comment. -# Doesn't apply if after a brace open or other single-line C comments. -nl_before_c_comment = 0 # unsigned number - -# The minimum number of newlines before a CPP comment. -# Doesn't apply if after a brace open or other CPP comments. -nl_before_cpp_comment = 0 # unsigned number - -# Whether to force a newline after a multi-line comment. -nl_after_multiline_comment = false # true/false - -# Whether to force a newline after a label's colon. -nl_after_label_colon = false # true/false - -# The number of newlines after '}' or ';' of a struct/enum/union definition. -nl_after_struct = 0 # unsigned number - -# The number of newlines before a class definition. -nl_before_class = 0 # unsigned number - -# The number of newlines after '}' or ';' of a class definition. -nl_after_class = 0 # unsigned number - -# The number of newlines before a namespace. -nl_before_namespace = 0 # unsigned number - -# The number of newlines after '{' of a namespace. This also adds newlines -# before the matching '}'. -# -# 0: Apply eat_blanks_after_open_brace or eat_blanks_before_close_brace if -# applicable, otherwise no change. -# -# Overrides eat_blanks_after_open_brace and eat_blanks_before_close_brace. -nl_inside_namespace = 0 # unsigned number - -# The number of newlines after '}' of a namespace. -nl_after_namespace = 0 # unsigned number - -# The number of newlines before an access specifier label. This also includes -# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count -# if after a brace open. -# -# 0: No change (default). -nl_before_access_spec = 0 # unsigned number - -# The number of newlines after an access specifier label. This also includes -# the Qt-specific 'signals:' and 'slots:'. Will not change the newline count -# if after a brace open. -# -# 0: No change (default). -# -# Overrides nl_typedef_blk_start and nl_var_def_blk_start. -nl_after_access_spec = 0 # unsigned number - -# The number of newlines between a function definition and the function -# comment, as in '// comment\n void foo() {...}'. -# -# 0: No change (default). -nl_comment_func_def = 0 # unsigned number - -# The number of newlines after a try-catch-finally block that isn't followed -# by a brace close. -# -# 0: No change (default). -nl_after_try_catch_finally = 0 # unsigned number - -# (C#) The number of newlines before and after a property, indexer or event -# declaration. -# -# 0: No change (default). -nl_around_cs_property = 0 # unsigned number - -# (C#) The number of newlines between the get/set/add/remove handlers. -# -# 0: No change (default). -nl_between_get_set = 0 # unsigned number - -# (C#) Add or remove newline between property and the '{'. -nl_property_brace = ignore # ignore/add/remove/force - -# Whether to remove blank lines after '{'. -eat_blanks_after_open_brace = false # true/false - -# Whether to remove blank lines before '}'. -eat_blanks_before_close_brace = false # true/false - -# How aggressively to remove extra newlines not in preprocessor. -# -# 0: No change (default) -# 1: Remove most newlines not handled by other config -# 2: Remove all newlines and reformat completely by config -nl_remove_extra_newlines = 0 # unsigned number - -# (Java) Add or remove newline after an annotation statement. Only affects -# annotations that are after a newline. -nl_after_annotation = ignore # ignore/add/remove/force - -# (Java) Add or remove newline between two annotations. -nl_between_annotation = ignore # ignore/add/remove/force - -# -# Positioning options -# - -# The position of arithmetic operators in wrapped expressions. -pos_arith = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of assignment in wrapped expressions. Do not affect '=' -# followed by '{'. -pos_assign = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of Boolean operators in wrapped expressions. -pos_bool = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of comparison operators in wrapped expressions. -pos_compare = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of conditional operators, as in the '?' and ':' of -# 'expr ? stmt : stmt', in wrapped expressions. -pos_conditional = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of the comma in wrapped expressions. -pos_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of the comma in enum entries. -pos_enum_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of the comma in the base class list if there is more than one -# line. Affects nl_class_init_args. -pos_class_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of the comma in the constructor initialization list. -# Related to nl_constr_colon, nl_constr_init_args and pos_constr_colon. -pos_constr_comma = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of trailing/leading class colon, between class and base class -# list. Affects nl_class_colon. -pos_class_colon = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# The position of colons between constructor and member initialization. -# Related to nl_constr_colon, nl_constr_init_args and pos_constr_comma. -pos_constr_colon = ignore # ignore/break/force/lead/trail/join/lead_break/lead_force/trail_break/trail_force - -# -# Line splitting options -# - -# Try to limit code width to N columns. -code_width = 0 # unsigned number - -# Whether to fully split long 'for' statements at semi-colons. -ls_for_split_full = false # true/false - -# Whether to fully split long function prototypes/calls at commas. -# The option ls_code_width has priority over the option ls_func_split_full. -ls_func_split_full = false # true/false - -# Whether to split lines as close to code_width as possible and ignore some -# groupings. -# The option ls_code_width has priority over the option ls_func_split_full. -ls_code_width = false # true/false - -# -# Code alignment options (not left column spaces/tabs) -# - -# Whether to keep non-indenting tabs. -align_keep_tabs = false # true/false - -# Whether to use tabs for aligning. -align_with_tabs = false # true/false - -# Whether to bump out to the next tab when aligning. -align_on_tabstop = false # true/false - -# Whether to right-align numbers. -align_number_right = false # true/false - -# Whether to keep whitespace not required for alignment. -align_keep_extra_space = false # true/false - -# Whether to align variable definitions in prototypes and functions. -align_func_params = false # true/false - -# The span for aligning parameter definitions in function on parameter name. -# -# 0: Don't align (default). -align_func_params_span = 0 # unsigned number - -# The threshold for aligning function parameter definitions. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_func_params_thresh = 0 # number - -# The gap for aligning function parameter definitions. -align_func_params_gap = 0 # unsigned number - -# The span for aligning constructor value. -# -# 0: Don't align (default). -align_constr_value_span = 0 # unsigned number - -# The threshold for aligning constructor value. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_constr_value_thresh = 0 # number - -# The gap for aligning constructor value. -align_constr_value_gap = 0 # unsigned number - -# Whether to align parameters in single-line functions that have the same -# name. The function names must already be aligned with each other. -align_same_func_call_params = false # true/false - -# The span for aligning function-call parameters for single line functions. -# -# 0: Don't align (default). -align_same_func_call_params_span = 0 # unsigned number - -# The threshold for aligning function-call parameters for single line -# functions. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_same_func_call_params_thresh = 0 # number - -# The span for aligning variable definitions. -# -# 0: Don't align (default). -align_var_def_span = 0 # unsigned number - -# How to consider (or treat) the '*' in the alignment of variable definitions. -# -# 0: Part of the type 'void * foo;' (default) -# 1: Part of the variable 'void *foo;' -# 2: Dangling 'void *foo;' -# Dangling: the '*' will not be taken into account when aligning. -align_var_def_star_style = 0 # unsigned number - -# How to consider (or treat) the '&' in the alignment of variable definitions. -# -# 0: Part of the type 'long & foo;' (default) -# 1: Part of the variable 'long &foo;' -# 2: Dangling 'long &foo;' -# Dangling: the '&' will not be taken into account when aligning. -align_var_def_amp_style = 0 # unsigned number - -# The threshold for aligning variable definitions. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_var_def_thresh = 0 # number - -# The gap for aligning variable definitions. -align_var_def_gap = 0 # unsigned number - -# Whether to align the colon in struct bit fields. -align_var_def_colon = false # true/false - -# The gap for aligning the colon in struct bit fields. -align_var_def_colon_gap = 0 # unsigned number - -# Whether to align any attribute after the variable name. -align_var_def_attribute = false # true/false - -# Whether to align inline struct/enum/union variable definitions. -align_var_def_inline = false # true/false - -# The span for aligning on '=' in assignments. -# -# 0: Don't align (default). -align_assign_span = 0 # unsigned number - -# The span for aligning on '=' in function prototype modifier. -# -# 0: Don't align (default). -align_assign_func_proto_span = 0 # unsigned number - -# The threshold for aligning on '=' in assignments. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_assign_thresh = 0 # number - -# How to apply align_assign_span to function declaration "assignments", i.e. -# 'virtual void foo() = 0' or '~foo() = {default|delete}'. -# -# 0: Align with other assignments (default) -# 1: Align with each other, ignoring regular assignments -# 2: Don't align -align_assign_decl_func = 0 # unsigned number - -# The span for aligning on '=' in enums. -# -# 0: Don't align (default). -align_enum_equ_span = 0 # unsigned number - -# The threshold for aligning on '=' in enums. -# Use a negative number for absolute thresholds. -# -# 0: no limit (default). -align_enum_equ_thresh = 0 # number - -# The span for aligning class member definitions. -# -# 0: Don't align (default). -align_var_class_span = 0 # unsigned number - -# The threshold for aligning class member definitions. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_var_class_thresh = 0 # number - -# The gap for aligning class member definitions. -align_var_class_gap = 0 # unsigned number - -# The span for aligning struct/union member definitions. -# -# 0: Don't align (default). -align_var_struct_span = 0 # unsigned number - -# The threshold for aligning struct/union member definitions. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_var_struct_thresh = 0 # number - -# The gap for aligning struct/union member definitions. -align_var_struct_gap = 0 # unsigned number - -# The span for aligning struct initializer values. -# -# 0: Don't align (default). -align_struct_init_span = 0 # unsigned number - -# The span for aligning single-line typedefs. -# -# 0: Don't align (default). -align_typedef_span = 0 # unsigned number - -# The minimum space between the type and the synonym of a typedef. -align_typedef_gap = 0 # unsigned number - -# How to align typedef'd functions with other typedefs. -# -# 0: Don't mix them at all (default) -# 1: Align the open parenthesis with the types -# 2: Align the function type name with the other type names -align_typedef_func = 0 # unsigned number - -# How to consider (or treat) the '*' in the alignment of typedefs. -# -# 0: Part of the typedef type, 'typedef int * pint;' (default) -# 1: Part of type name: 'typedef int *pint;' -# 2: Dangling: 'typedef int *pint;' -# Dangling: the '*' will not be taken into account when aligning. -align_typedef_star_style = 0 # unsigned number - -# How to consider (or treat) the '&' in the alignment of typedefs. -# -# 0: Part of the typedef type, 'typedef int & intref;' (default) -# 1: Part of type name: 'typedef int &intref;' -# 2: Dangling: 'typedef int &intref;' -# Dangling: the '&' will not be taken into account when aligning. -align_typedef_amp_style = 0 # unsigned number - -# The span for aligning comments that end lines. -# -# 0: Don't align (default). -align_right_cmt_span = 0 # unsigned number - -# Minimum number of columns between preceding text and a trailing comment in -# order for the comment to qualify for being aligned. Must be non-zero to have -# an effect. -align_right_cmt_gap = 0 # unsigned number - -# If aligning comments, whether to mix with comments after '}' and #endif with -# less than three spaces before the comment. -align_right_cmt_mix = false # true/false - -# Whether to only align trailing comments that are at the same brace level. -align_right_cmt_same_level = false # true/false - -# Minimum column at which to align trailing comments. Comments which are -# aligned beyond this column, but which can be aligned in a lesser column, -# may be "pulled in". -# -# 0: Ignore (default). -align_right_cmt_at_col = 0 # unsigned number - -# The span for aligning function prototypes. -# -# 0: Don't align (default). -align_func_proto_span = 0 # unsigned number - -# The threshold for aligning function prototypes. -# Use a negative number for absolute thresholds. -# -# 0: No limit (default). -align_func_proto_thresh = 0 # number - -# Minimum gap between the return type and the function name. -align_func_proto_gap = 0 # unsigned number - -# Whether to align function prototypes on the 'operator' keyword instead of -# what follows. -align_on_operator = false # true/false - -# Whether to mix aligning prototype and variable declarations. If true, -# align_var_def_XXX options are used instead of align_func_proto_XXX options. -align_mix_var_proto = false # true/false - -# Whether to align single-line functions with function prototypes. -# Uses align_func_proto_span. -align_single_line_func = false # true/false - -# Whether to align the open brace of single-line functions. -# Requires align_single_line_func=true. Uses align_func_proto_span. -align_single_line_brace = false # true/false - -# Gap for align_single_line_brace. -align_single_line_brace_gap = 0 # unsigned number - -# (OC) The span for aligning Objective-C message specifications. -# -# 0: Don't align (default). -align_oc_msg_spec_span = 0 # unsigned number - -# Whether to align macros wrapped with a backslash and a newline. This will -# not work right if the macro contains a multi-line comment. -align_nl_cont = false # true/false - -# Whether to align macro functions and variables together. -align_pp_define_together = false # true/false - -# The span for aligning on '#define' bodies. -# -# =0: Don't align (default) -# >0: Number of lines (including comments) between blocks -align_pp_define_span = 0 # unsigned number - -# The minimum space between label and value of a preprocessor define. -align_pp_define_gap = 0 # unsigned number - -# Whether to align lines that start with '<<' with previous '<<'. -# -# Default: true -align_left_shift = true # true/false - -# Whether to align text after 'asm volatile ()' colons. -align_asm_colon = false # true/false - -# (OC) Span for aligning parameters in an Objective-C message call -# on the ':'. -# -# 0: Don't align. -align_oc_msg_colon_span = 0 # unsigned number - -# (OC) Whether to always align with the first parameter, even if it is too -# short. -align_oc_msg_colon_first = false # true/false - -# (OC) Whether to align parameters in an Objective-C '+' or '-' declaration -# on the ':'. -align_oc_decl_colon = false # true/false - -# -# Comment modification options -# - -# Try to wrap comments at N columns. -cmt_width = 0 # unsigned number - -# How to reflow comments. -# -# 0: No reflowing (apart from the line wrapping due to cmt_width) (default) -# 1: No touching at all -# 2: Full reflow -cmt_reflow_mode = 0 # unsigned number - -# Whether to convert all tabs to spaces in comments. If false, tabs in -# comments are left alone, unless used for indenting. -cmt_convert_tab_to_spaces = false # true/false - -# Whether to apply changes to multi-line comments, including cmt_width, -# keyword substitution and leading chars. -# -# Default: true -cmt_indent_multi = true # true/false - -# Whether to group c-comments that look like they are in a block. -cmt_c_group = false # true/false - -# Whether to put an empty '/*' on the first line of the combined c-comment. -cmt_c_nl_start = false # true/false - -# Whether to add a newline before the closing '*/' of the combined c-comment. -cmt_c_nl_end = false # true/false - -# Whether to change cpp-comments into c-comments. -cmt_cpp_to_c = false # true/false - -# Whether to group cpp-comments that look like they are in a block. Only -# meaningful if cmt_cpp_to_c=true. -cmt_cpp_group = false # true/false - -# Whether to put an empty '/*' on the first line of the combined cpp-comment -# when converting to a c-comment. -# -# Requires cmt_cpp_to_c=true and cmt_cpp_group=true. -cmt_cpp_nl_start = false # true/false - -# Whether to add a newline before the closing '*/' of the combined cpp-comment -# when converting to a c-comment. -# -# Requires cmt_cpp_to_c=true and cmt_cpp_group=true. -cmt_cpp_nl_end = false # true/false - -# Whether to put a star on subsequent comment lines. -cmt_star_cont = false # true/false - -# The number of spaces to insert at the start of subsequent comment lines. -cmt_sp_before_star_cont = 0 # unsigned number - -# The number of spaces to insert after the star on subsequent comment lines. -cmt_sp_after_star_cont = 0 # unsigned number - -# For multi-line comments with a '*' lead, remove leading spaces if the first -# and last lines of the comment are the same length. -# -# Default: true -cmt_multi_check_last = true # true/false - -# For multi-line comments with a '*' lead, remove leading spaces if the first -# and last lines of the comment are the same length AND if the length is -# bigger as the first_len minimum. -# -# Default: 4 -cmt_multi_first_len_minimum = 4 # unsigned number - -# Path to a file that contains text to insert at the beginning of a file if -# the file doesn't start with a C/C++ comment. If the inserted text contains -# '$(filename)', that will be replaced with the current file's name. -cmt_insert_file_header = "" # string - -# Path to a file that contains text to insert at the end of a file if the -# file doesn't end with a C/C++ comment. If the inserted text contains -# '$(filename)', that will be replaced with the current file's name. -cmt_insert_file_footer = "" # string - -# Path to a file that contains text to insert before a function definition if -# the function isn't preceded by a C/C++ comment. If the inserted text -# contains '$(function)', '$(javaparam)' or '$(fclass)', these will be -# replaced with, respectively, the name of the function, the javadoc '@param' -# and '@return' stuff, or the name of the class to which the member function -# belongs. -cmt_insert_func_header = "" # string - -# Path to a file that contains text to insert before a class if the class -# isn't preceded by a C/C++ comment. If the inserted text contains '$(class)', -# that will be replaced with the class name. -cmt_insert_class_header = "" # string - -# Path to a file that contains text to insert before an Objective-C message -# specification, if the method isn't preceded by a C/C++ comment. If the -# inserted text contains '$(message)' or '$(javaparam)', these will be -# replaced with, respectively, the name of the function, or the javadoc -# '@param' and '@return' stuff. -cmt_insert_oc_msg_header = "" # string - -# Whether a comment should be inserted if a preprocessor is encountered when -# stepping backwards from a function name. -# -# Applies to cmt_insert_oc_msg_header, cmt_insert_func_header and -# cmt_insert_class_header. -cmt_insert_before_preproc = false # true/false - -# Whether a comment should be inserted if a function is declared inline to a -# class definition. -# -# Applies to cmt_insert_func_header. -# -# Default: true -cmt_insert_before_inlines = true # true/false - -# Whether a comment should be inserted if the function is a class constructor -# or destructor. -# -# Applies to cmt_insert_func_header. -cmt_insert_before_ctor_dtor = false # true/false - -# -# Code modifying options (non-whitespace) -# - -# Add or remove braces on a single-line 'do' statement. -mod_full_brace_do = ignore # ignore/add/remove/force - -# Add or remove braces on a single-line 'for' statement. -mod_full_brace_for = ignore # ignore/add/remove/force - -# (Pawn) Add or remove braces on a single-line function definition. -mod_full_brace_function = ignore # ignore/add/remove/force - -# Add or remove braces on a single-line 'if' statement. Braces will not be -# removed if the braced statement contains an 'else'. -mod_full_brace_if = ignore # ignore/add/remove/force - -# Whether to enforce that all blocks of an 'if'/'else if'/'else' chain either -# have, or do not have, braces. If true, braces will be added if any block -# needs braces, and will only be removed if they can be removed from all -# blocks. -# -# Overrides mod_full_brace_if. -mod_full_brace_if_chain = false # true/false - -# Whether to add braces to all blocks of an 'if'/'else if'/'else' chain. -# If true, mod_full_brace_if_chain will only remove braces from an 'if' that -# does not have an 'else if' or 'else'. -mod_full_brace_if_chain_only = false # true/false - -# Add or remove braces on single-line 'while' statement. -mod_full_brace_while = ignore # ignore/add/remove/force - -# Add or remove braces on single-line 'using ()' statement. -mod_full_brace_using = ignore # ignore/add/remove/force - -# Don't remove braces around statements that span N newlines -mod_full_brace_nl = 0 # unsigned number - -# Whether to prevent removal of braces from 'if'/'for'/'while'/etc. blocks -# which span multiple lines. -# -# Affects: -# mod_full_brace_for -# mod_full_brace_if -# mod_full_brace_if_chain -# mod_full_brace_if_chain_only -# mod_full_brace_while -# mod_full_brace_using -# -# Does not affect: -# mod_full_brace_do -# mod_full_brace_function -mod_full_brace_nl_block_rem_mlcond = false # true/false - -# Add or remove unnecessary parenthesis on 'return' statement. -mod_paren_on_return = ignore # ignore/add/remove/force - -# (Pawn) Whether to change optional semicolons to real semicolons. -mod_pawn_semicolon = false # true/false - -# Whether to fully parenthesize Boolean expressions in 'while' and 'if' -# statement, as in 'if (a && b > c)' => 'if (a && (b > c))'. -mod_full_paren_if_bool = false # true/false - -# Whether to remove superfluous semicolons. -mod_remove_extra_semicolon = false # true/false - -# If a function body exceeds the specified number of newlines and doesn't have -# a comment after the close brace, a comment will be added. -mod_add_long_function_closebrace_comment = 0 # unsigned number - -# If a namespace body exceeds the specified number of newlines and doesn't -# have a comment after the close brace, a comment will be added. -mod_add_long_namespace_closebrace_comment = 0 # unsigned number - -# If a class body exceeds the specified number of newlines and doesn't have a -# comment after the close brace, a comment will be added. -mod_add_long_class_closebrace_comment = 0 # unsigned number - -# If a switch body exceeds the specified number of newlines and doesn't have a -# comment after the close brace, a comment will be added. -mod_add_long_switch_closebrace_comment = 0 # unsigned number - -# If an #ifdef body exceeds the specified number of newlines and doesn't have -# a comment after the #endif, a comment will be added. -mod_add_long_ifdef_endif_comment = 0 # unsigned number - -# If an #ifdef or #else body exceeds the specified number of newlines and -# doesn't have a comment after the #else, a comment will be added. -mod_add_long_ifdef_else_comment = 0 # unsigned number - -# Whether to take care of the case by the mod_sort_xx options. -mod_sort_case_sensitive = false # true/false - -# Whether to sort consecutive single-line 'import' statements. -mod_sort_import = false # true/false - -# (C#) Whether to sort consecutive single-line 'using' statements. -mod_sort_using = false # true/false - -# Whether to sort consecutive single-line '#include' statements (C/C++) and -# '#import' statements (Objective-C). Be aware that this has the potential to -# break your code if your includes/imports have ordering dependencies. -mod_sort_include = false # true/false - -# Whether to move a 'break' that appears after a fully braced 'case' before -# the close brace, as in 'case X: { ... } break;' => 'case X: { ... break; }'. -mod_move_case_break = false # true/false - -# Add or remove braces around a fully braced case statement. Will only remove -# braces if there are no variable declarations in the block. -mod_case_brace = ignore # ignore/add/remove/force - -# Whether to remove a void 'return;' that appears as the last statement in a -# function. -mod_remove_empty_return = false # true/false - -# Add or remove the comma after the last value of an enumeration. -mod_enum_last_comma = ignore # ignore/add/remove/force - -# (OC) Whether to organize the properties. If true, properties will be -# rearranged according to the mod_sort_oc_property_*_weight factors. -mod_sort_oc_properties = false # true/false - -# (OC) Weight of a class property modifier. -mod_sort_oc_property_class_weight = 0 # number - -# (OC) Weight of 'atomic' and 'nonatomic'. -mod_sort_oc_property_thread_safe_weight = 0 # number - -# (OC) Weight of 'readwrite' when organizing properties. -mod_sort_oc_property_readwrite_weight = 0 # number - -# (OC) Weight of a reference type specifier ('retain', 'copy', 'assign', -# 'weak', 'strong') when organizing properties. -mod_sort_oc_property_reference_weight = 0 # number - -# (OC) Weight of getter type ('getter=') when organizing properties. -mod_sort_oc_property_getter_weight = 0 # number - -# (OC) Weight of setter type ('setter=') when organizing properties. -mod_sort_oc_property_setter_weight = 0 # number - -# (OC) Weight of nullability type ('nullable', 'nonnull', 'null_unspecified', -# 'null_resettable') when organizing properties. -mod_sort_oc_property_nullability_weight = 0 # number - -# -# Preprocessor options -# - -# Add or remove indentation of preprocessor directives inside #if blocks -# at brace level 0 (file-level). -pp_indent = ignore # ignore/add/remove/force - -# Whether to indent #if/#else/#endif at the brace level. If false, these are -# indented from column 1. -pp_indent_at_level = false # true/false - -# Specifies the number of columns to indent preprocessors per level -# at brace level 0 (file-level). If pp_indent_at_level=false, also specifies -# the number of columns to indent preprocessors per level -# at brace level > 0 (function-level). -# -# Default: 1 -pp_indent_count = 1 # unsigned number - -# Add or remove space after # based on pp_level of #if blocks. -pp_space = ignore # ignore/add/remove/force - -# Sets the number of spaces per level added with pp_space. -pp_space_count = 0 # unsigned number - -# The indent for '#region' and '#endregion' in C# and '#pragma region' in -# C/C++. Negative values decrease indent down to the first column. -pp_indent_region = 0 # number - -# Whether to indent the code between #region and #endregion. -pp_region_indent_code = false # true/false - -# If pp_indent_at_level=true, sets the indent for #if, #else and #endif when -# not at file-level. Negative values decrease indent down to the first column. -# -# =0: Indent preprocessors using output_tab_size -# >0: Column at which all preprocessors will be indented -pp_indent_if = 0 # number - -# Whether to indent the code between #if, #else and #endif. -pp_if_indent_code = false # true/false - -# Whether to indent '#define' at the brace level. If false, these are -# indented from column 1. -pp_define_at_level = false # true/false - -# Whether to ignore the '#define' body while formatting. -pp_ignore_define_body = false # true/false - -# Whether to indent case statements between #if, #else, and #endif. -# Only applies to the indent of the preprocesser that the case statements -# directly inside of. -# -# Default: true -pp_indent_case = true # true/false - -# Whether to indent whole function definitions between #if, #else, and #endif. -# Only applies to the indent of the preprocesser that the function definition -# is directly inside of. -# -# Default: true -pp_indent_func_def = true # true/false - -# Whether to indent extern C blocks between #if, #else, and #endif. -# Only applies to the indent of the preprocesser that the extern block is -# directly inside of. -# -# Default: true -pp_indent_extern = true # true/false - -# Whether to indent braces directly inside #if, #else, and #endif. -# Only applies to the indent of the preprocesser that the braces are directly -# inside of. -# -# Default: true -pp_indent_brace = true # true/false - -# -# Sort includes options -# - -# The regex for include category with priority 0. -include_category_0 = "" # string - -# The regex for include category with priority 1. -include_category_1 = "" # string - -# The regex for include category with priority 2. -include_category_2 = "" # string - -# -# Use or Do not Use options -# - -# true: indent_func_call_param will be used (default) -# false: indent_func_call_param will NOT be used -# -# Default: true -use_indent_func_call_param = true # true/false - -# The value of the indentation for a continuation line is calculated -# differently if the statement is: -# - a declaration: your case with QString fileName ... -# - an assignment: your case with pSettings = new QSettings( ... -# -# At the second case the indentation value might be used twice: -# - at the assignment -# - at the function call (if present) -# -# To prevent the double use of the indentation value, use this option with the -# value 'true'. -# -# true: indent_continue will be used only once -# false: indent_continue will be used every time (default) -use_indent_continue_only_once = false # true/false - -# The value might be used twice: -# - at the assignment -# - at the opening brace -# -# To prevent the double use of the indentation value, use this option with the -# value 'true'. -# -# true: indentation will be used only once -# false: indentation will be used every time (default) -indent_cpp_lambda_only_once = true # true/false - -# Whether sp_after_angle takes precedence over sp_inside_fparen. This was the -# historic behavior, but is probably not the desired behavior, so this is off -# by default. -use_sp_after_angle_always = false # true/false - -# Whether to apply special formatting for Qt SIGNAL/SLOT macros. Essentially, -# this tries to format these so that they match Qt's normalized form (i.e. the -# result of QMetaObject::normalizedSignature), which can slightly improve the -# performance of the QObject::connect call, rather than how they would -# otherwise be formatted. -# -# See options_for_QT.cpp for details. -# -# Default: true -use_options_overriding_for_qt_macros = true # true/false - -# -# Warn levels - 1: error, 2: warning (default), 3: note -# - -# (C#) Warning is given if doing tab-to-\t replacement and we have found one -# in a C# verbatim string literal. -# -# Default: 2 -warn_level_tabs_found_in_verbatim_string_literals = 2 # unsigned number - -# Meaning of the settings: -# Ignore - do not do any changes -# Add - makes sure there is 1 or more space/brace/newline/etc -# Force - makes sure there is exactly 1 space/brace/newline/etc, -# behaves like Add in some contexts -# Remove - removes space/brace/newline/etc -# -# -# - Token(s) can be treated as specific type(s) with the 'set' option: -# `set tokenType tokenString [tokenString...]` -# -# Example: -# `set BOOL __AND__ __OR__` -# -# tokenTypes are defined in src/token_enum.h, use them without the -# 'CT_' prefix: 'CT_BOOL' => 'BOOL' -# -# -# - Token(s) can be treated as type(s) with the 'type' option. -# `type tokenString [tokenString...]` -# -# Example: -# `type int c_uint_8 Rectangle` -# -# This can also be achieved with `set TYPE int c_uint_8 Rectangle` -# -# -# To embed whitespace in tokenStrings use the '\' escape character, or quote -# the tokenStrings. These quotes are supported: "'` -# -# -# - Support for the auto detection of languages through the file ending can be -# added using the 'file_ext' command. -# `file_ext langType langString [langString..]` -# -# Example: -# `file_ext CPP .ch .cxx .cpp.in` -# -# langTypes are defined in uncrusify_types.h in the lang_flag_e enum, use -# them without the 'LANG_' prefix: 'LANG_CPP' => 'CPP' -# -# -# - Custom macro-based indentation can be set up using 'macro-open', -# 'macro-else' and 'macro-close'. -# `(macro-open | macro-else | macro-close) tokenString` -# -# Example: -# `macro-open BEGIN_TEMPLATE_MESSAGE_MAP` -# `macro-open BEGIN_MESSAGE_MAP` -# `macro-close END_MESSAGE_MAP` -# -# -# option(s) with 'not default' value: 0 -# - -file_ext CPP .ch .cxx .cpp.in .cc .hh - From 72ecccee5781ec0b57b2860f677b8a64db43cda7 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 12:19:34 -0600 Subject: [PATCH 246/350] convert to logWarning format --- src/libexpr/parser.y | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 3ae7bbafd..49fb8ad37 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -717,7 +717,11 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; } catch (FileTransferError & e) { - printError("warning: Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second); + logWarning( + ErrorInfo { + .name = "Entry download", + .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + }); res = { false, "" }; } } else { @@ -727,7 +731,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl else { logWarning( ErrorInfo { - .name = "Search path not found", + .name = "Entry not found", .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; From 960d4362ed2215f87a596e0a994e39aa28680db1 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 13:54:18 -0600 Subject: [PATCH 247/350] hint only --- src/error-demo/error-demo.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 8f484431c..82e9f0580 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -149,5 +149,13 @@ int main() .errPos = Pos(problem_file, 40, 13) }}); + // Error with only hint and name.. + logError( + ErrorInfo { .name = "error name", + .hint = hintfmt("hint %1%", "only"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + return 0; } From ecbb8e9c0a4374b26defde2cd10932d633affed3 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 12 May 2020 14:41:30 -0600 Subject: [PATCH 248/350] no blank line if no LOC --- Makefile | 3 ++- src/error-demo/error-demo.cc | 1 + src/libutil/error.cc | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0ba011e2a..05c4bdc8a 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,8 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk + tests/plugins/local.mk \ + src/error-demo/local.mk -include Makefile.config diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 82e9f0580..0216092e3 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -1,5 +1,6 @@ #include "logging.hh" #include "nixexpr.hh" +#include "util.hh" #include #include diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e4f45a2d3..0765a2945 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -182,7 +182,7 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) } // lines of code. - if (einfo.nixCode.has_value()) { + if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) { printCodeLines(out, prefix, *einfo.nixCode); out << prefix << std::endl; } From bfca5fc395bdaf5823413e1a1679f1b0b6db29dd Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 13 May 2020 09:52:36 -0600 Subject: [PATCH 249/350] change status messages to info level --- src/libstore/build.cc | 2 +- src/libstore/gc.cc | 12 ++++++------ src/libstore/local-store.cc | 18 +++++++++--------- src/libutil/logging.cc | 6 +++++- src/libutil/util.cc | 13 +------------ src/nix-env/user-env.cc | 2 +- src/nix/make-content-addressable.cc | 2 +- src/nix/upgrade-nix.cc | 2 +- 8 files changed, 25 insertions(+), 32 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0359b10ac..961110d58 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4862,7 +4862,7 @@ void Worker::waitForInput() if (!waitingForAWhile.empty()) { useTimeout = true; if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); + printInfo("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; timeout = std::max(1L, (long) std::chrono::duration_cast( diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e99423257..e529416b7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -41,7 +41,7 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError("waiting for the big garbage collector lock..."); + printInfo("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -231,7 +231,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError("removing stale temporary roots file '%1%'", path); + printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -751,7 +751,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError("finding garbage collector roots..."); + printInfo("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); @@ -803,9 +803,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError("deleting garbage..."); + printInfo("deleting garbage..."); else - printError("determining live/dead paths..."); + printInfo("determining live/dead paths..."); try { @@ -872,7 +872,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError("deleting unused links..."); + printInfo("deleting unused links..."); removeUnusedLinks(state); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63a825af0..59d551032 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -152,7 +152,7 @@ LocalStore::LocalStore(const Params & params) globalLock = openLockFile(globalLockPath.c_str(), true); if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); + printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -183,7 +183,7 @@ LocalStore::LocalStore(const Params & params) "please upgrade Nix to version 1.11 first."); if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); + printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltWrite, true); } @@ -261,7 +261,7 @@ LocalStore::~LocalStore() } if (future.valid()) { - printError("waiting for auto-GC to finish on exit..."); + printInfo("waiting for auto-GC to finish on exit..."); future.get(); } @@ -1210,7 +1210,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printError(format("reading the Nix store...")); + printInfo(format("reading the Nix store...")); bool errors = false; @@ -1251,7 +1251,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) }); if (repair) { if (unlink(linkPath.c_str()) == 0) - printError("removed link '%s'", linkPath); + printInfo("removed link '%s'", linkPath); else throw SysError("removing corrupt link '%s'", linkPath); } else { @@ -1294,14 +1294,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError("fixing missing hash on '%s'", printStorePath(i)); + printInfo("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError("updating size field on '%s' to %s", printStorePath(i), current.second); + printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1360,7 +1360,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, } if (canInvalidate) { - printError("path '%s' disappeared, removing from database...", pathS); + printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { @@ -1431,7 +1431,7 @@ static void makeMutable(const Path & path) void LocalStore::upgradeStore7() { if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); + printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 736ce604e..071a7ec7e 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -251,7 +251,11 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); + logError( + ErrorInfo { + .name = "Json log message", + .hint = hintfmt("bad log message from builder: %s", e.what()) + }); } return true; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ac7c2967b..dcf89ff69 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -857,7 +857,7 @@ int Pid::kill() #if __FreeBSD__ || __APPLE__ if (errno != EPERM || ::kill(pid, 0) != 0) #endif - printError((SysError("killing process %d", pid).msg())); + logError(SysError("killing process %d", pid).info()); } return wait(); @@ -1447,17 +1447,6 @@ string base64Decode(const string & s) } -// void callFailure(const std::function & failure, std::exception_ptr exc) -// { -// try { -// failure(exc); -// } catch (std::exception & e) { -// printError("uncaught exception: %s", e.what()); -// abort(); -// } -// } - - static Sync> windowSize{{0, 0}}; diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 2484b9759..f804b77a0 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -146,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Path lockTokenCur = optimisticLockProfile(profile); if (lockToken != lockTokenCur) { - printError("profile '%1%' changed while we were busy; restarting", profile); + printInfo("profile '%1%' changed while we were busy; restarting", profile); return false; } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 8803461f4..0da4fa4b4 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -85,7 +85,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON info.ca = makeFixedOutputCA(true, info.narHash); if (!json) - printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 9e7ebcd9c..fdf94e5a3 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -98,7 +98,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } - printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); } /* Return the profile in which Nix is installed. */ From c79d4addab7ea476f551bc8727e4d1f27adcef31 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 13 May 2020 10:02:18 -0600 Subject: [PATCH 250/350] consistent capitalization --- src/build-remote/build-remote.cc | 2 +- src/libstore/builtins/buildenv.cc | 4 ++-- src/libstore/filetransfer.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 4 ++-- src/libstore/sqlite.cc | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 0d6859596..0caa97fa4 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -202,7 +202,7 @@ static int _main(int argc, char * * argv) auto msg = chomp(drainFD(5, false)); logError( ErrorInfo { - .name = "remote build", + .name = "Remote build", .hint = hintfmt("cannot build on '%s': %s%s", bestMachine->storeUri, e.what(), (msg.empty() ? "" : ": " + msg)) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 64085fcc9..6c493ed77 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -24,7 +24,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (e.errNo == ENOTDIR) { logWarning( ErrorInfo { - .name = "Create Links - Directory", + .name = "Create links - directory", .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) }); return; @@ -47,7 +47,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (e.errNo == ENOENT || e.errNo == ENOTDIR) { logWarning( ErrorInfo { - .name = "Create Links - Skipping Symlink", + .name = "Create links - skipping symlink", .hint = hintfmt("skipping dangling symlink '%s'", dstFile) }); continue; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 3285edded..89588d662 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -601,7 +601,7 @@ struct curlFileTransfer : public FileTransfer } catch (std::exception & e) { logError( ErrorInfo { - .name = "download", + .name = "File transfer", .hint = hintfmt("unexpected error in download thread: %s", e.what()) }); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e529416b7..d53d458e9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -132,7 +132,7 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, logWarning( ErrorInfo { - .name = "GC Root", + .name = "GC root", .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " "therefore, '%2%' might be removed by the garbage collector", gcRoot, printStorePath(storePath)) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 59d551032..194eefc78 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1244,7 +1244,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (hash != link.name) { logError( ErrorInfo { - .name = "Invalid Hash", + .name = "Invalid hash", .hint = hintfmt( "link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash) @@ -1283,7 +1283,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { logError( ErrorInfo { - .name = "Invalid Hash - Path Modified", + .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", printStorePath(i), info->narHash.to_string(), current.first.to_string()) }); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 9f6112183..c9ecca5a8 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -132,7 +132,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { logWarning( ErrorInfo { - .name = "Suspicious File", + .name = "Suspicious file", .hint = hintfmt("skipping suspicious writable file '%1%'", path) }); return; @@ -200,7 +200,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (st.st_size != stLink.st_size) { logWarning( ErrorInfo { - .name = "Corrupted Link", + .name = "Corrupted link", .hint = hintfmt("removing corrupted link '%1%'", linkPath) }); unlink(linkPath.c_str()); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 082c54005..34ef5e57b 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -205,7 +205,7 @@ void handleSQLiteBusy(SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; logWarning( - ErrorInfo { .name = "sqlite busy", + ErrorInfo { .name = "Sqlite busy", .hint = hintfmt(e.what()) }); } From d44bac1d9232374bf823ed24f5a5ae693f89fb7d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 13 May 2020 12:39:45 -0600 Subject: [PATCH 251/350] remove error-demo from Makefile again --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 05c4bdc8a..0ba011e2a 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,7 @@ makefiles = \ misc/upstart/local.mk \ doc/manual/local.mk \ tests/local.mk \ - tests/plugins/local.mk \ - src/error-demo/local.mk + tests/plugins/local.mk -include Makefile.config From ef9dd9f9bc18abc9761812e30a26008c99a26166 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 13 May 2020 15:56:39 -0600 Subject: [PATCH 252/350] formatting and a few minor changes --- src/libexpr/parser.y | 2 - src/libexpr/primops.cc | 59 ++++++++++++--------------- src/libexpr/primops/context.cc | 2 - src/libexpr/primops/fetchMercurial.cc | 1 - src/libstore/build.cc | 40 ++++++++---------- src/libstore/gc.cc | 1 - src/libstore/local-store.cc | 2 +- src/libstore/sqlite.cc | 2 +- src/libstore/sqlite.hh | 2 +- src/libutil/types.hh | 8 ++-- src/nix-daemon/nix-daemon.cc | 4 +- src/nix-env/nix-env.cc | 6 +-- src/nix-store/nix-store.cc | 2 +- 13 files changed, 54 insertions(+), 77 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 49fb8ad37..0417a3c21 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -427,7 +427,6 @@ expr_simple .hint = hintfmt("URL literals are disabled"), .nixCode = NixCode { .errPos = CUR_POS } }); - $$ = new ExprString(data->symbols.create($1)); } | '(' expr ')' { $$ = $2; } @@ -696,7 +695,6 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - throw ThrownError( ErrorInfo { .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 657ce3001..d21c49813 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -99,7 +99,6 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args path, e.path), .nixCode = NixCode { .errPos = pos } }); - } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -179,7 +178,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value ErrorInfo { .hint = hintfmt( "cannot import '%1%', since path '%2%' is not valid", - path, e.path), + path, e.path), .nixCode = NixCode { .errPos = pos } }); } @@ -199,8 +198,8 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value if (message) throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message); else - throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected" - , sym, path); + throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", + sym, path); } (func)(state, v); @@ -233,8 +232,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) } catch (InvalidPathError & e) { throw EvalError( ErrorInfo { - .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid" - , program, e.path), + .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + program, e.path), .nixCode = NixCode { .errPos = pos } });} @@ -625,8 +624,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .hint = hintfmt("duplicate derivation output '%1%'", j), .nixCode = NixCode { .errPos = posDrvName } }); - - /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because @@ -638,7 +635,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .hint = hintfmt("invalid derivation output name 'drv'" ), .nixCode = NixCode { .errPos = posDrvName } }); - outputs.insert(j); } if (outputs.empty()) @@ -1236,7 +1232,6 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value path), .nixCode = NixCode { .errPos = *attr.pos } }); - } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { @@ -1322,7 +1317,6 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) .hint = hintfmt("attribute '%1%' missing", attr), .nixCode = NixCode { .errPos = pos } }); - // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value, pos); @@ -1407,7 +1401,6 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), .nixCode = NixCode { .errPos = pos } }); - string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); @@ -1419,7 +1412,6 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), .nixCode = NixCode { .errPos = pos } }); - v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } } @@ -1498,7 +1490,6 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args .nixCode = NixCode { .errPos = pos } }); - if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); return; @@ -2038,16 +2029,16 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError( - ErrorInfo { - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + ErrorInfo { + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), .nixCode = NixCode { .errPos = pos } }); } else { - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -2111,19 +2102,19 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError( - ErrorInfo { - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); - } else { - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); - } + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError( + ErrorInfo { + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } else { + throw EvalError( + ErrorInfo { + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); + } } } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 768936453..7f895fc01 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -151,7 +151,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .hint = hintfmt("Context key '%s' is not a store path", i.name), .nixCode = NixCode { .errPos = *i.pos } }); - if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(i.name)); state.forceAttrs(*i.value, *i.pos); @@ -170,7 +169,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), .nixCode = NixCode { .errPos = *i.pos } }); - } context.insert("=" + string(i.name)); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 1e8b9d4a0..bb008ba6b 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -43,7 +43,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), .nixCode = NixCode { .errPos = *attr.pos } }); - } if (url.empty()) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 961110d58..64bc9c4b8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -497,8 +497,8 @@ void handleDiffHook( printError(chomp(diffRes.second)); } catch (Error & error) { ErrorInfo ei = error.info(); - string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : ""); - ei.hint = std::optional(hintfmt("diff hook execution failed: %s", prevhint)); + ei.hint = hintfmt("diff hook execution failed: %s", + (error.info().hint.has_value() ? error.info().hint->str() : "")); logError(ei); } } @@ -1151,7 +1151,7 @@ void DerivationGoal::loadDerivation() if (nrFailed != 0) { logError( - ErrorInfo { + ErrorInfo { .name = "missing derivation during build", .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) }); @@ -1306,13 +1306,12 @@ void DerivationGoal::repairClosure() for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; logError( - ErrorInfo { + ErrorInfo { .name = "Corrupt path in closure", .hint = hintfmt( "found corrupted or missing path '%s' in the output closure of '%s'", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) }); - auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); @@ -1347,7 +1346,7 @@ void DerivationGoal::inputsRealised() if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); logError( - ErrorInfo { + ErrorInfo { .name = "Dependencies could not be built", .hint = hintfmt( "cannot build derivation '%s': %s dependencies couldn't be built", @@ -1811,7 +1810,7 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { logError( - ErrorInfo { + ErrorInfo { .name = "Build hook died", .hint = hintfmt( "build hook died unexpectedly: %s", @@ -3676,7 +3675,7 @@ void DerivationGoal::registerOutputs() if (!outputRewrites.empty()) { logWarning( ErrorInfo { - .name = "Rewriting hashes", + .name = "Rewriting hashes", .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) }); @@ -3856,7 +3855,7 @@ void DerivationGoal::registerOutputs() throw NotDeterministic(hint); logError( - ErrorInfo { + ErrorInfo { .name = "Output determinism error", .hint = hint }); @@ -4124,7 +4123,7 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { logError( - ErrorInfo { + ErrorInfo { .name = "Max log size exceeded", .hint = hintfmt( "%1% killed after writing more than %2% bytes of log output", @@ -4447,9 +4446,9 @@ void SubstitutionGoal::tryNext() { logWarning( ErrorInfo { - .name = "Invalid path signature", + .name = "Invalid path signature", .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) + sub->getUri(), worker.store.printStorePath(storePath)) }); tryNext(); return; @@ -4809,7 +4808,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0 == settings.maxBuildJobs) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds"); assert(!awake.empty()); @@ -4933,7 +4932,7 @@ void Worker::waitForInput() after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { logError( - ErrorInfo { + ErrorInfo { .name = "Silent build timeout", .hint = hintfmt( "%1% timed out after %2% seconds of silence", @@ -4948,7 +4947,7 @@ void Worker::waitForInput() after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { logError( - ErrorInfo { + ErrorInfo { .name = "Build timeout", .hint = hintfmt( "%1% timed out after %2% seconds", @@ -4969,9 +4968,6 @@ void Worker::waitForInput() } - - - unsigned int Worker::exitStatus() { /* @@ -5015,9 +5011,9 @@ bool Worker::pathContentsGood(const StorePath & path) res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path.clone(), res); - if (!res) + if (!res) logError( - ErrorInfo { + ErrorInfo { .name = "Corrupted path", .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) }); @@ -5136,8 +5132,4 @@ void LocalStore::repairPath(const StorePath & path) } - } - - - diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d53d458e9..92f9328c3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -129,7 +129,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, if (settings.checkRootReachability) { auto roots = findRoots(false); if (roots[storePath.clone()].count(gcRoot) == 0) - logWarning( ErrorInfo { .name = "GC root", diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 194eefc78..6f88a3e32 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -504,7 +504,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe chown(path.c_str(), geteuid(), getegid()) == -1) #endif throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); + path, geteuid()); } if (S_ISDIR(st.st_mode)) { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 34ef5e57b..16cdb6619 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -196,7 +196,7 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e) { static std::atomic lastWarned{0}; diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index ccfac48d8..dd81ab051 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -99,7 +99,7 @@ MakeError(SQLiteBusy, SQLiteError); [[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); -void handleSQLiteBusy(SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e); /* Convenience function for retrying a SQLite transaction when the database is busy. */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 4d783b564..66072e8f7 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -15,13 +15,13 @@ using std::set; using std::vector; using std::string; -typedef list Strings; -typedef set StringSet; -typedef std::map StringMap; +typedef list Strings; +typedef set StringSet; +typedef std::map StringMap; /* Paths are just strings. */ -typedef std::string Path; +typedef string Path; typedef list Paths; typedef set PathSet; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 731ab22e5..582c78d14 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -248,8 +248,8 @@ static void daemonLoop(char * * argv) return; } catch (Error & error) { ErrorInfo ei = error.info(); - string prevhint = (error.info().hint.has_value() ? error.info().hint->str() : ""); - ei.hint = std::optional(hintfmt("error processing connection: %1%", prevhint)); + ei.hint = std::optional(hintfmt("error processing connection: %1%", + (error.info().hint.has_value() ? error.info().hint->str() : ""))); logError(ei); } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0f99f5fe1..bbce5dcda 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -69,7 +69,7 @@ typedef void (* Operation) (Globals & globals, static string needArg(Strings::iterator & i, Strings & args, const string & arg) { - if (i == args.end()) throw UsageError( "'%1%' requires an argument", arg); + if (i == args.end()) throw UsageError("'%1%' requires an argument", arg); return *i++; } @@ -151,7 +151,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) { struct stat st; if (stat(path.c_str(), &st) == -1) - throw SysError("getting inon about '%1%'", path); + throw SysError("getting information about '%1%'", path); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -1233,7 +1233,7 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) throw Error("no generation older than the current (%1%) exists", - curGen); + curGen); else throw Error("generation %1% does not exist", dstGen); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9b5cceccf..77deea5a7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -731,7 +731,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { logError( ErrorInfo { - .name = "Hash match error", + .name = "Hash mismatch", .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), From 183dd28266e0292116da6cfbc3e2643577adbfbe Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 14 May 2020 17:00:54 +0300 Subject: [PATCH 253/350] Don't lock a user while doing remote builds --- src/libstore/build.cc | 88 ++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a8c1cd565..f7fc23b35 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -507,6 +507,7 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; + bool isEnabled; string user; uid_t uid; gid_t gid; @@ -524,7 +525,7 @@ public: bool findFreeUser(); - bool enabled() { return uid != 0; } + bool enabled() { return isEnabled; } }; @@ -535,9 +536,11 @@ UserLock::UserLock() createDirs(settings.nixStateDir + "/userpool"); /* Mark that user is not enabled by default */ uid = 0; + isEnabled = false; } bool UserLock::findFreeUser() { + if (enabled()) return true; /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -597,6 +600,7 @@ bool UserLock::findFreeUser() { supplementaryGIDs.resize(ngroups); #endif + isEnabled = true; return true; } } @@ -931,6 +935,7 @@ private: void closureRepaired(); void inputsRealised(); void tryToBuild(); + void tryLocalBuild(); void buildDone(); /* Is the build hook willing to perform the build? */ @@ -1002,6 +1007,8 @@ private: Goal::amDone(result); } + void started(); + void done(BuildResult::Status status, const string & msg = ""); StorePathSet exportReferences(const StorePathSet & storePaths); @@ -1389,35 +1396,24 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); +} void DerivationGoal::tryToBuild() { trace("trying to build"); - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); - - if (!buildUser->enabled()) { - if (!buildUser->findFreeUser()) { - debug("waiting for build users"); - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. If we can't acquire the lock, then continue; hopefully some other @@ -1464,20 +1460,6 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); - auto started = [&]() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); - }; - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { @@ -1510,6 +1492,34 @@ void DerivationGoal::tryToBuild() return; } + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + debug("waiting for build users"); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + try { /* Okay, we have to build. */ From 4daccb279c547b90b0c17c8a64825f04abdf0f7e Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 14 May 2020 10:28:17 -0600 Subject: [PATCH 254/350] formatting --- src/libexpr/primops.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d21c49813..9b9cbfc2e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -767,7 +767,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .nixCode = NixCode { .errPos = posDrvName } }); - /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) throw EvalError( @@ -776,7 +775,6 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .nixCode = NixCode { .errPos = posDrvName } }); - if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") From 19694aa213961daa5cbe2263bfaca53dc068a40c Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 14 May 2020 12:28:18 -0600 Subject: [PATCH 255/350] fix compile errors --- src/libutil/error.cc | 16 ++++++++++++++++ src/libutil/error.hh | 16 ++-------------- src/libutil/types.hh | 2 +- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 0765a2945..59628d875 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -3,6 +3,7 @@ #include #include #include "serialise.hh" +#include namespace nix { @@ -16,6 +17,21 @@ BaseError & BaseError::addPrefix(const FormatOrString & fs) return *this; } +const string& BaseError::calcWhat() const +{ + if (what_.has_value()) + return *what_; + else { + err.name = sname(); + + std::ostringstream oss; + oss << err; + what_ = oss.str(); + + return *what_; + } +} + std::optional ErrorInfo::programName = std::nullopt; std::ostream& operator<<(std::ostream &os, const hintformat &hf) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 19c6e35a1..b374c2780 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -88,20 +88,8 @@ protected: mutable ErrorInfo err; mutable std::optional what_; - const string& calcWhat() const - { - if (what_.has_value()) - return *what_; - else { - err.name = sname(); - - std::ostringstream oss; - oss << err; - what_ = oss.str(); - - return *what_; - } - } + const string& calcWhat() const; + public: unsigned int status = 1; // exit status diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 66072e8f7..89ae108f9 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -6,7 +6,7 @@ #include #include #include - +#include namespace nix { From 546b179d0a89e9f27a02e92004da0f8f08e5041a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Fri, 15 May 2020 10:06:14 +0200 Subject: [PATCH 256/350] actions: use latest OS --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87997414d..7feefc855 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: tests: strategy: matrix: - os: [ubuntu-18.04, macos] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From e223eeac09c6468db80ee0497c84960b4ec73e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 May 2020 08:40:55 +0100 Subject: [PATCH 257/350] Remove -j option from simple-build-testing By default Nix/NixOS already set a reasonable default `max-jobs = auto` so we don't need to mention it in this tutorial. The option is still documented in other parts of the documentation if users ever stumble over this. Fixes https://github.com/NixOS/nix/issues/2531 --- doc/manual/expressions/simple-building-testing.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index 7326a3e76..ce0a1636d 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'make). -If you have a system with multiple CPUs, you may want to have -Nix build different derivations in parallel (insofar as possible). -Just pass the option , where -N is the maximum number of jobs to be run -in parallel, or set. Typically this should be the number of -CPUs. -
From 5ef64f05e6e493d5fe69c87127328f68500b9e50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 May 2020 15:50:29 +0200 Subject: [PATCH 258/350] Cleanup --- src/libstore/build.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4f2f38d63..a2b16f95c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -507,10 +507,10 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; - bool isEnabled; + bool isEnabled = false; string user; - uid_t uid; - gid_t gid; + uid_t uid = 0; + gid_t gid = 0; std::vector supplementaryGIDs; public: @@ -534,9 +534,6 @@ UserLock::UserLock() { assert(settings.buildUsersGroup != ""); createDirs(settings.nixStateDir + "/userpool"); - /* Mark that user is not enabled by default */ - uid = 0; - isEnabled = false; } bool UserLock::findFreeUser() { From a73a820a5d73e0c595ff807a752f76ef1184ca2d Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Wed, 20 May 2020 16:27:53 +0200 Subject: [PATCH 259/350] Add unit testes for url.cc This adds tests for - parseURL - percentDecode - decodeQuery --- src/libutil/tests/tests.cc | 6 +- src/libutil/tests/url.cc | 266 +++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 src/libutil/tests/url.cc diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 5abfe2c9c..8e77ccbe1 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -5,6 +5,8 @@ namespace nix { +/* ----------- tests for util.hh ------------------------------------------------*/ + /* ---------------------------------------------------------------------------- * absPath * --------------------------------------------------------------------------*/ @@ -15,6 +17,9 @@ namespace nix { ASSERT_EQ(p, "/"); } + + + TEST(absPath, turnsEmptyPathIntoCWD) { char cwd[PATH_MAX+1]; auto p = absPath(""); @@ -581,5 +586,4 @@ namespace nix { ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); } - } diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc new file mode 100644 index 000000000..80646ad3e --- /dev/null +++ b/src/libutil/tests/url.cc @@ -0,0 +1,266 @@ +#include "url.hh" +#include + +namespace nix { + +/* ----------- tests for url.hh --------------------------------------------------*/ + + string print_map(std::map m) { + std::map::iterator it; + string s = "{ "; + for (it = m.begin(); it != m.end(); ++it) { + s += "{ "; + s += it->first; + s += " = "; + s += it->second; + s += " } "; + } + s += "}"; + return s; + } + + + std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { + return os << "\n" + << "url: " << p.url << "\n" + << "base: " << p.base << "\n" + << "scheme: " << p.scheme << "\n" + << "authority: " << p.authority.value() << "\n" + << "path: " << p.path << "\n" + << "query: " << print_map(p.query) << "\n" + << "fragment: " << p.fragment << "\n"; + } + + TEST(parseURL, parsesSimpleHttpUrl) { + auto s = "http://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpsUrl) { + auto s = "https://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) { + auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) { + auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "field", "value" } }, + .fragment = "?foo=bar#", + }; + + ASSERT_EQ(parsed, expected); + } + + + TEST(parseURL, parseIPv4Address) { + auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://127.0.0.1:8080/file.tar.gz", + .base = "https://127.0.0.1:8080/file.tar.gz", + .scheme = "http", + .authority = "127.0.0.1:8080", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseIPv6Address) { + auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .scheme = "http", + .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parseEmptyQueryParams) { + auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&"; + auto parsed = parseURL(s); + ASSERT_EQ(parsed.query, (StringMap) { }); + } + + TEST(parseURL, parseUserPassword) { + auto s = "http://user:pass@www.example.org:8080/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://user:pass@www.example.org/file.tar.gz", + .base = "http://user:pass@www.example.org/file.tar.gz", + .scheme = "http", + .authority = "user:pass@www.example.org:8080", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseFileURLWithQueryAndFragment) { + auto s = "file:///none/of/your/business"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "", + .base = "", + .scheme = "file", + .authority = "", + .path = "/none/of/your/business", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parsedUrlsIsEqualToItself) { + auto s = "http://www.example.org/file.tar.gz"; + auto url = parseURL(s); + + ASSERT_TRUE(url == url); + } + + TEST(parseURL, parseFTPUrl) { + auto s = "ftp://ftp.nixos.org/downloads/nixos.iso"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "ftp://ftp.nixos.org/downloads/nixos.iso", + .base = "ftp://ftp.nixos.org/downloads/nixos.iso", + .scheme = "ftp", + .authority = "ftp.nixos.org", + .path = "/downloads/nixos.iso", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesAnythingInUriFormat) { + auto s = "whatever://github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) { + auto s = "whatever:github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, emptyStringIsInvalidURL) { + ASSERT_THROW(parseURL(""), Error); + } + + /* ---------------------------------------------------------------------------- + * decodeQuery + * --------------------------------------------------------------------------*/ + + TEST(decodeQuery, emptyStringYieldsEmptyMap) { + auto d = decodeQuery(""); + ASSERT_EQ(d, (StringMap) { }); + } + + TEST(decodeQuery, simpleDecode) { + auto d = decodeQuery("yi=one&er=two"); + ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } })); + } + + TEST(decodeQuery, decodeUrlEncodedArgs) { + auto d = decodeQuery("arg=%3D%3D%40%3D%3D"); + ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } })); + } + + TEST(decodeQuery, decodeArgWithEmptyValue) { + auto d = decodeQuery("arg="); + ASSERT_EQ(d, ((StringMap) { { "arg", ""} })); + } + + /* ---------------------------------------------------------------------------- + * percentDecode + * --------------------------------------------------------------------------*/ + + TEST(percentDecode, decodesUrlEncodedString) { + string s = "==@=="; + string d = percentDecode("%3D%3D%40%3D%3D"); + ASSERT_EQ(d, s); + } + + TEST(percentDecode, multipleDecodesAreIdempotent) { + string once = percentDecode("%3D%3D%40%3D%3D"); + string twice = percentDecode(once); + + ASSERT_EQ(once, twice); + } + + TEST(percentDecode, trailingPercent) { + string s = "==@==%"; + string d = percentDecode("%3D%3D%40%3D%3D%25"); + + ASSERT_EQ(d, s); + } + +} From c8cb558849da8ef88f15cc2a70c570f1f5013a30 Mon Sep 17 00:00:00 2001 From: Krzysztof Gogolewski Date: Thu, 21 May 2020 19:29:13 +0200 Subject: [PATCH 260/350] documentation: avoid unquoted URLs --- doc/manual/command-ref/conf-file.xml | 2 +- doc/manual/expressions/advanced-attributes.xml | 4 ++-- doc/manual/expressions/builtins.xml | 4 ++-- doc/manual/expressions/expression-syntax.xml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 1820598e5..1fa74a143 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -386,7 +386,7 @@ false. builtins.fetchurl { - url = https://example.org/foo-1.2.3.tar.xz; + url = "https://example.org/foo-1.2.3.tar.xz"; sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; } diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index 372d03de7..5759ff50e 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" ... ]; fetchurl { - url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } @@ -189,7 +189,7 @@ fetchurl { fetchurl { - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index f71a8f3be..6709c4081 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -349,7 +349,7 @@ stdenv.mkDerivation { … } with import (fetchTarball { - url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; + url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz"; sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; }) {}; @@ -1406,7 +1406,7 @@ stdenv.mkDerivation { "; src = fetchurl { - url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml index 42b9dca36..a3de20713 100644 --- a/doc/manual/expressions/expression-syntax.xml +++ b/doc/manual/expressions/expression-syntax.xml @@ -15,7 +15,7 @@ stdenv.mkDerivation { name = "hello-2.1.1"; builder = ./builder.sh; src = fetchurl { - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; From 0726ad5825f60e543d9cf535c62673685adbf5c8 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 15 Dec 2019 16:43:43 +0100 Subject: [PATCH 261/350] install: configure and bootstrap synthetic.conf on darwin Starting macOS 10.15 /nix can't be creasted directly anymore due to the readonly filesystem, but synthetic.conf was introduced to enable creating mountpoints or symlinks for special usecases like package managers. --- release.nix | 12 +++- scripts/create-darwin-volume.sh | 107 ++++++++++++++++++++++++++++ scripts/install-nix-from-closure.sh | 93 +++++++++++++++--------- 3 files changed, 176 insertions(+), 36 deletions(-) create mode 100755 scripts/create-darwin-volume.sh diff --git a/release.nix b/release.nix index f5729cee3..2cc4bb4c0 100644 --- a/release.nix +++ b/release.nix @@ -177,10 +177,10 @@ let } '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} @@ -195,6 +195,7 @@ let # SC1090: Don't worry about not being able to find # $nix/etc/profile.d/nix.sh shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh shellcheck $TMPDIR/install-darwin-multi-user.sh shellcheck $TMPDIR/install-systemd-multi-user.sh @@ -210,6 +211,7 @@ let fi chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh chmod +x $TMPDIR/install-darwin-multi-user.sh chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user @@ -222,11 +224,15 @@ let --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ $(cat ${installerClosureInfo}/store-paths) ''); diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh new file mode 100755 index 000000000..f3e0d46f1 --- /dev/null +++ b/scripts/create-darwin-volume.sh @@ -0,0 +1,107 @@ +#!/bin/sh +set -e + +root_disks() { + diskutil list -plist / +} + +apfs_volumes_for() { + disk=$1 + diskutil apfs list -plist "$disk" +} + +disk_identifier() { + xpath "/plist/dict/key[text()='WholeDisks']/following-sibling::array[1]/string/text()" 2>/dev/null +} + +volume_get() { + key=$1 i=$2 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null +} + +find_nix_volume() { + disk=$1 + i=1 + volumes=$(apfs_volumes_for "$disk") + while true; do + name=$(echo "$volumes" | volume_get "Name" "$i") + if [ -z "$name" ]; then + break + fi + case "$name" in + [Nn]ix*) + echo "$name" + break + ;; + esac + i=$((i+1)) + done +} + +test_fstab() { + grep -q "/nix" /etc/fstab 2>/dev/null +} + +test_synthetic_conf() { + grep -q "^nix" /etc/synthetic.conf 2>/dev/null +} + +test_nix() { + test -d "/nix" +} + +main() { + ( + echo "" + echo " ------------------------------------------------------------------ " + echo " | This installer will create a volume for the nix store and |" + echo " | configure it to mount at /nix. Follow these steps to uninstall. |" + echo " ------------------------------------------------------------------ " + echo "" + echo " 1. Remove the entry from fstab using 'sudo vifs'" + echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" + echo " 3. Delete /etc/synthetic.conf" + echo "" + ) >&2 + + if [ -L "/nix" ]; then + echo "error: /nix is a symlink, please remove it or edit synthetic.conf (requires reboot)" >&2 + echo " /nix -> $(readlink "/nix")" >&2 + exit 2 + fi + + if ! test_synthetic_conf; then + echo "Configuring /etc/synthetic.conf..." >&2 + echo nix | sudo tee /etc/synthetic.conf + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + fi + + if ! test_nix; then + echo "Creating mountpoint for /nix..." >&2 + sudo mkdir /nix + fi + + disk=$(root_disks | disk_identifier) + volume=$(find_nix_volume "$disk") + if [ -z "$volume" ]; then + echo "Creating a Nix Store volume..." >&2 + sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix + volume="Nix Store" + else + echo "Using existing '$volume' volume" >&2 + fi + + if ! test_fstab; then + echo "Configuring /etc/fstab..." >&2 + label=$(echo "$volume" | sed 's/ /\\040/g') + printf "\$a\nLABEL=%s /nix apfs rw\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + fi + + echo "The following options can be enabled to disable spotlight indexing" >&2 + echo "of the volume, which might be desirable." >&2 + echo "" >&2 + echo " $ mdutil -i off /nix" >&2 + echo "" >&2 +} + +main "$@" diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index e06530ddf..34be7ee6a 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -40,44 +40,62 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then fi INSTALL_MODE=no-daemon - +CREATE_DARWIN_VOLUME=0 # handle the command line flags -while [ "x${1:-}" != "x" ]; do - if [ "x${1:-}" = "x--no-daemon" ]; then - INSTALL_MODE=no-daemon - elif [ "x${1:-}" = "x--daemon" ]; then - INSTALL_MODE=daemon - elif [ "x${1:-}" = "x--no-channel-add" ]; then - NIX_INSTALLER_NO_CHANNEL_ADD=1 - elif [ "x${1:-}" = "x--no-modify-profile" ]; then - NIX_INSTALLER_NO_MODIFY_PROFILE=1 - elif [ "x${1:-}" != "x" ]; then - ( - echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]" +while [ $# -gt 0 ]; do + case $1 in + --daemon) + INSTALL_MODE=daemon;; + --no-daemon) + INSTALL_MODE=no-daemon;; + --no-channel-add) + NIX_INSTALLER_NO_CHANNEL_ADD=1;; + --no-modify-profile) + NIX_INSTALLER_NO_MODIFY_PROFILE=1;; + --create-volume) + CREATE_DARWIN_VOLUME=1;; + *) + ( + echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]" - echo "Choose installation method." - echo "" - echo " --daemon: Installs and configures a background daemon that manages the store," - echo " providing multi-user support and better isolation for local builds." - echo " Both for security and reproducibility, this method is recommended if" - echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" - echo "" - echo " --no-daemon: Simple, single-user installation that does not require root and is" - echo " trivial to uninstall." - echo " (default)" - echo "" - echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." - echo "" - echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" - echo " is installed by default." - echo "" - ) >&2 - exit - fi + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." + echo "" + echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" + echo " is installed by default." + echo "" + ) >&2 + + if [ "$(uname -s)" = "Darwin" ]; then + ( + echo " --create-volume: Create an APFS volume for the store and create the /nix" + echo " mountpoint for it using synthetic.conf." + echo " (required on macOS >=10.15)" + echo " See https://nixos.org/nix/manual/#sect-darwin-apfs-volume" + echo "" + ) >&2 + fi + exit;; + esac shift done +if [ "$(uname -s)" = "Darwin" ] && [ "$CREATE_DARWIN_VOLUME" = 1 ]; then + printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' + "$self/create-darwin-volume.sh" +fi + if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' exec "$self/install-multi-user" @@ -95,6 +113,15 @@ if ! [ -e $dest ]; then echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 + if [ "$(uname -s)" = "Darwin" ]; then + ( + echo "" + echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." + echo "Use --create-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume." + echo "" + ) >&2 + fi exit 1 fi fi From 10202628b911980f05fc2c9460995af30f1bcbf3 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 15 Dec 2019 21:11:14 +0100 Subject: [PATCH 262/350] install: also configure ~/.zshenv The default login shell for users on macOS 10.15 changed from bash to zsh. So while generally nonstandard we need to configure it to make nix function out of the box on macOS. --- scripts/install-nix-from-closure.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 34be7ee6a..88275d1f0 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -197,6 +197,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done + for i in .zshenv .zshrc; do + fn="$HOME/$i" + if [ -w "$fn" ]; then + if ! grep -q "$p" "$fn"; then + echo "modifying $fn..." >&2 + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + fi + added=1 + break + fi + done fi if [ -z "$added" ]; then From 083bb3bbfcdccebd06bde81a66f158d51ed6e455 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 13 Feb 2020 12:57:35 +0100 Subject: [PATCH 263/350] install: show macOS 10.15 message with --daemon --- scripts/create-darwin-volume.sh | 8 ++++---- scripts/install-nix-from-closure.sh | 28 ++++++++++++++++------------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index f3e0d46f1..d6fb44f43 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -1,8 +1,8 @@ #!/bin/sh set -e -root_disks() { - diskutil list -plist / +root_disk() { + diskutil info -plist / } apfs_volumes_for() { @@ -11,7 +11,7 @@ apfs_volumes_for() { } disk_identifier() { - xpath "/plist/dict/key[text()='WholeDisks']/following-sibling::array[1]/string/text()" 2>/dev/null + xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null } volume_get() { @@ -81,7 +81,7 @@ main() { sudo mkdir /nix fi - disk=$(root_disks | disk_identifier) + disk=$(root_disk | disk_identifier) volume=$(find_nix_volume "$disk") if [ -z "$volume" ]; then echo "Creating a Nix Store volume..." >&2 diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 88275d1f0..7d32bf92e 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -91,9 +91,22 @@ while [ $# -gt 0 ]; do shift done -if [ "$(uname -s)" = "Darwin" ] && [ "$CREATE_DARWIN_VOLUME" = 1 ]; then - printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' - "$self/create-darwin-volume.sh" +if [ "$(uname -s)" = "Darwin" ]; then + if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then + printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' + "$self/create-darwin-volume.sh" + fi + + info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) + if ! [ -e $dest ] && [ -n "$info" ]; then + ( + echo "" + echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." + echo "Use --create-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume." + echo "" + ) >&2 + fi fi if [ "$INSTALL_MODE" = "daemon" ]; then @@ -113,15 +126,6 @@ if ! [ -e $dest ]; then echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 - if [ "$(uname -s)" = "Darwin" ]; then - ( - echo "" - echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use --create-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume." - echo "" - ) >&2 - fi exit 1 fi fi From ee89b7797d4ec1db6dad9df5fb3bb8cc2f05de12 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 17 Jan 2020 23:27:29 +0100 Subject: [PATCH 264/350] manual: add apfs volume section --- doc/manual/installation/installing-binary.xml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 3f57f47b5..86cbce6bf 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -136,6 +136,109 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist +
+ APFS Volume Installation + + + The root filesystem is read-only as of macOS 10.15 Catalina, all writable + paths to a separate data volume. This means creating or writing to /nix + is not allowed. While changing the default prefix would be possible, it's + a very intrusive change that has side effects we want to avoid for now. + + + + For common writable locations firmlinks where introduced, + described by Apple as a "bi-directional wormhole" between two filesystems. + Essentially a bind mount for APFS volumes. However this is (currently) not + user configurable and only available for paths like /Users. + + + + For special cases like NFS mount points or package manager roots synthetic.conf(5) + provides a mechanism for some limited, user-controlled file-creation at /. + This only applies on a reboot, but apfs.util can be used + to trigger the creation (not deletion) of new entries. + + + +alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + + + + + + The simplest solution is creating a symlink with /etc/synthetic.conf + to the data volume. (not recommended) + + + +nix /System/Volumes/Data/nix + + + +alice$ ls -l / +lrwxr-xr-x 1 root wheel 25 Jan 1 2019 nix -> /System/Volumes/Data/nix + + + + However builds that detect or resolve this symlink will leak the canonical + location or even fail in certain cases, making this approach undesirable. + + + + + + An empty directory can also be created using /etc/synthetic.conf, + this won't be writable but can be used as a mount point. And with + APFS it's relatively easy to create an separate + volume for nix instead. + + + +nix + + + +alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix +alice$ mount +/dev/disk1s6 on /nix (apfs, local, journaled) + + + + This does make the installation more complicated, requiring both + /etc/synthetic.conf as well as /etc/fstab + + + +# +# Warning - this file should only be modified with vifs(8) +# +# Failure to do so is unsupported and may be destructive. +# +LABEL=Nix\040Store /nix apfs rw + + + + On macOS volumes are also mounted quite late, launchd services or other + things that start during login will start before our volume is mounted. + For these cases eg. wait4path must be used for + things that depend on /nix. + + + + This new volume also won't be encrypted by default, and enabling is + only possible interactively? + + + +diskutil apfs enableFileVault /nix -user disk + + + + + +
+
Installing a pinned Nix version from a URL From caface1980344347f2a50701ec133e98e6094c8c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 5 Mar 2020 23:12:52 +0100 Subject: [PATCH 265/350] install: hide the store volume on darwin --- scripts/create-darwin-volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index d6fb44f43..a1a67acea 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -94,7 +94,7 @@ main() { if ! test_fstab; then echo "Configuring /etc/fstab..." >&2 label=$(echo "$volume" | sed 's/ /\\040/g') - printf "\$a\nLABEL=%s /nix apfs rw\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs fi echo "The following options can be enabled to disable spotlight indexing" >&2 From 04f597c3f4d0ac8b8a677c798642329a5a6768e3 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 7 Mar 2020 12:01:40 +0100 Subject: [PATCH 266/350] install: improve output and error handling --- doc/manual/installation/installing-binary.xml | 3 ++- scripts/create-darwin-volume.sh | 17 ++++++++++++++--- scripts/install-nix-from-closure.sh | 7 ++++--- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 86cbce6bf..7f4bba3a0 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -199,6 +199,7 @@ nix +alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix alice$ mount /dev/disk1s6 on /nix (apfs, local, journaled) @@ -231,7 +232,7 @@ LABEL=Nix\040Store /nix apfs rw -diskutil apfs enableFileVault /nix -user disk +alice$ diskutil apfs enableFileVault /nix -user disk diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index a1a67acea..e2b49c9a0 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -73,12 +73,22 @@ main() { if ! test_synthetic_conf; then echo "Configuring /etc/synthetic.conf..." >&2 echo nix | sudo tee /etc/synthetic.conf - /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + if ! test_synthetic_conf; then + echo "error: failed to configure synthetic.conf" >&2 + exit 1 + fi fi if ! test_nix; then echo "Creating mountpoint for /nix..." >&2 - sudo mkdir /nix + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true + if ! test_nix; then + sudo mkdir -p /nix 2>/dev/null || true + fi + if ! test_nix; then + echo "error: failed to bootstrap /nix, a reboot might be required" >&2 + exit 1 + fi fi disk=$(root_disk | disk_identifier) @@ -97,10 +107,11 @@ main() { printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs fi + echo "" >&2 echo "The following options can be enabled to disable spotlight indexing" >&2 echo "of the volume, which might be desirable." >&2 echo "" >&2 - echo " $ mdutil -i off /nix" >&2 + echo " $ sudo mdutil -i off /nix" >&2 echo "" >&2 } diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 7d32bf92e..2f291ed4c 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -82,7 +82,7 @@ while [ $# -gt 0 ]; do echo " --create-volume: Create an APFS volume for the store and create the /nix" echo " mountpoint for it using synthetic.conf." echo " (required on macOS >=10.15)" - echo " See https://nixos.org/nix/manual/#sect-darwin-apfs-volume" + echo " See https://nixos.org/nix/manual/#sect-apfs-volume-installation" echo "" ) >&2 fi @@ -102,10 +102,11 @@ if [ "$(uname -s)" = "Darwin" ]; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use --create-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-darwin-apfs-volume." + echo "Use sh <(curl https://nixos.org/nix/install) --create-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" echo "" ) >&2 + exit 1 fi fi From bc24c09968bb35fce599151f86d123cb5984f727 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Wed, 18 Mar 2020 20:34:46 +0100 Subject: [PATCH 267/350] install: make synthetic.conf and fstab checks stricter --- scripts/create-darwin-volume.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index e2b49c9a0..ea4133444 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -39,11 +39,15 @@ find_nix_volume() { } test_fstab() { - grep -q "/nix" /etc/fstab 2>/dev/null + grep -q "/nix apfs rw" /etc/fstab 2>/dev/null +} + +test_nix_symlink() { + [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null } test_synthetic_conf() { - grep -q "^nix" /etc/synthetic.conf 2>/dev/null + grep -q "^nix$" /etc/synthetic.conf 2>/dev/null } test_nix() { @@ -60,12 +64,12 @@ main() { echo "" echo " 1. Remove the entry from fstab using 'sudo vifs'" echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" - echo " 3. Delete /etc/synthetic.conf" + echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" echo "" ) >&2 - if [ -L "/nix" ]; then - echo "error: /nix is a symlink, please remove it or edit synthetic.conf (requires reboot)" >&2 + if test_nix_symlink; then + echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 echo " /nix -> $(readlink "/nix")" >&2 exit 2 fi From 33865752960c9a2ff28eb9024f20d2103918685c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 26 Mar 2020 20:14:08 +0100 Subject: [PATCH 268/350] manual: clarify volume creation section --- doc/manual/installation/installing-binary.xml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 7f4bba3a0..c11ba9cce 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -141,13 +141,14 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist The root filesystem is read-only as of macOS 10.15 Catalina, all writable - paths to a separate data volume. This means creating or writing to /nix - is not allowed. While changing the default prefix would be possible, it's - a very intrusive change that has side effects we want to avoid for now. + paths were moved to a separate data volume. This means creating or writing + to /nix is not allowed. While changing the default prefix + would be possible, it's a very intrusive change that has side effects we want to + avoid for now. - For common writable locations firmlinks where introduced, + For common writable locations firmlinks were introduced, described by Apple as a "bi-directional wormhole" between two filesystems. Essentially a bind mount for APFS volumes. However this is (currently) not user configurable and only available for paths like /Users. @@ -156,8 +157,10 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist For special cases like NFS mount points or package manager roots synthetic.conf(5) provides a mechanism for some limited, user-controlled file-creation at /. - This only applies on a reboot, but apfs.util can be used - to trigger the creation (not deletion) of new entries. + This only applies at boot time, however apfs.util can be used + to trigger the creation (not deletion) of new entries without a reboot. + It would be ideal if this could create firmlinks, however a symlink or mountpoint + are the only options. From 477d7c2d07e146c91950401b8b9d9380ce6787e5 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 26 Mar 2020 21:51:13 +0100 Subject: [PATCH 269/350] installer: refuse apfs volume creation when FileVault is enabled --- doc/manual/installation/installing-binary.xml | 6 +++-- scripts/create-darwin-volume.sh | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index c11ba9cce..498248662 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -230,8 +230,10 @@ LABEL=Nix\040Store /nix apfs rw - This new volume also won't be encrypted by default, and enabling is - only possible interactively? + This new volume also won't be encrypted by default, and enabling it + requires extra setup. For machines with a T2 chip + all data is already entrypted at rest, older hardware won't even when + FileVault is enabled for the rest of the system. diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index ea4133444..a0da85f43 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -14,7 +14,12 @@ disk_identifier() { xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null } -volume_get() { +volume_list_true() { + key=$1 t=$2 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null +} + +volume_get_string() { key=$1 i=$2 xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null } @@ -24,7 +29,7 @@ find_nix_volume() { i=1 volumes=$(apfs_volumes_for "$disk") while true; do - name=$(echo "$volumes" | volume_get "Name" "$i") + name=$(echo "$volumes" | volume_get_string "Name" "$i") if [ -z "$name" ]; then break fi @@ -54,6 +59,12 @@ test_nix() { test -d "/nix" } +test_filevault() { + disk=$1 + apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true || return + ! sudo xartutil --list >/dev/null 2>/dev/null +} + main() { ( echo "" @@ -99,6 +110,13 @@ main() { volume=$(find_nix_volume "$disk") if [ -z "$volume" ]; then echo "Creating a Nix Store volume..." >&2 + + if test_filevault "$disk"; then + echo "error: FileVault detected, refusing to create unencrypted volume" >&2 + echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" >&2 + exit 1 + fi + sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix volume="Nix Store" else From 2b0a81d92d28994374465c44c79f020d5e044700 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 14 May 2020 21:59:10 -0500 Subject: [PATCH 270/350] focus on golden-path covering most scenarios This should handle installation scenarios we can handle with anything resembling confidence. Goal is approximating the existing setup--not enforcing a best-practice... Approaches (+ installer-handled, - manual) and configs each covers: + no change needed; /nix OK on boot volume: All pre-Catalina (regardless of T2 or FileVault use) + create new unencrypted volume: Catalina, pre-T2, no FileVault + create new encrypted-at-rest volume: Catalina, pre-T2, FileVault Catalina, T2, no FileVault - require user to pre-create encrypted volume Catalina, T2, FileVault --- doc/manual/installation/installing-binary.xml | 350 +++++++++++++----- scripts/create-darwin-volume.sh | 77 +++- scripts/install-nix-from-closure.sh | 19 +- 3 files changed, 331 insertions(+), 115 deletions(-) diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 498248662..8d548f0ea 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -6,16 +6,30 @@ Installing a Binary Distribution -If you are using Linux or macOS, the easiest way to install Nix -is to run the following command: + + If you are using Linux or macOS versions up to 10.14 (Mojave), the + easiest way to install Nix is to run the following command: + $ sh <(curl https://nixos.org/nix/install) -As of Nix 2.1.0, the Nix installer will always default to creating a -single-user installation, however opting in to the multi-user -installation is highly recommended. + + If you're using macOS 10.15 (Catalina) or newer, consult + the macOS installation instructions + before installing. + + + + As of Nix 2.1.0, the Nix installer will always default to creating a + single-user installation, however opting in to the multi-user + installation is highly recommended. +
@@ -36,7 +50,7 @@ run this under your usual user account, not as root. The script will invoke sudo to create /nix if it doesn’t already exist. If you don’t have sudo, you should manually create -/nix first as root, e.g.: +/nix first as root, e.g.: $ mkdir /nix @@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst .bash_profile, .bash_login and .profile to source ~/.nix-profile/etc/profile.d/nix.sh. You can set -the NIX_INSTALLER_NO_MODIFY_PROFILE environment +the NIX_INSTALLER_NO_MODIFY_PROFILE environment variable before executing the install script to disable this behaviour. @@ -81,12 +95,10 @@ $ rm -rf /nix You can instruct the installer to perform a multi-user installation on your system: - - - sh <(curl https://nixos.org/nix/install) --daemon - + sh <(curl https://nixos.org/nix/install) --daemon + The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. @@ -136,82 +148,253 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist
-
- APFS Volume Installation +
+ macOS Installation - The root filesystem is read-only as of macOS 10.15 Catalina, all writable - paths were moved to a separate data volume. This means creating or writing - to /nix is not allowed. While changing the default prefix - would be possible, it's a very intrusive change that has side effects we want to - avoid for now. + Starting with macOS 10.15 (Catalina), the root filesystem is read-only. + This means /nix can no longer live on your system + volume, and that you'll need a workaround to install Nix. - For common writable locations firmlinks were introduced, - described by Apple as a "bi-directional wormhole" between two filesystems. - Essentially a bind mount for APFS volumes. However this is (currently) not - user configurable and only available for paths like /Users. + The recommended approach, which creates an unencrypted APFS volume + for your Nix store and a "synthetic" empty directory to mount it + over at /nix, is least likely to impair Nix + or your system. + + With all separate-volume approaches, it's possible something on + your system (particularly daemons/services and restored apps) may + need access to your Nix store before the volume is mounted. Adding + additional encryption makes this more likely. + + - For special cases like NFS mount points or package manager roots synthetic.conf(5) - provides a mechanism for some limited, user-controlled file-creation at /. - This only applies at boot time, however apfs.util can be used - to trigger the creation (not deletion) of new entries without a reboot. - It would be ideal if this could create firmlinks, however a symlink or mountpoint - are the only options. + If you're using a recent Mac with a + T2 chip, + your drive will still be encrypted at rest (in which case "unencrypted" + is a bit of a misnomer). To use this approach, just install Nix with: - -alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B - + $ sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume - - - - The simplest solution is creating a symlink with /etc/synthetic.conf - to the data volume. (not recommended) - + + If you don't like the sound of this, you'll want to weigh the + other approaches and tradeoffs detailed in this section. + - -nix /System/Volumes/Data/nix - + + Eventual solutions? + + All of the known workarounds have drawbacks, but we hope + better solutions will be available in the future. Some that + we have our eye on are: + + + + + A true firmlink would enable the Nix store to live on the + primary data volume without the build problems caused by + the symlink approach. End users cannot currently + create true firmlinks. + + + + + If the Nix store volume shared FileVault encryption + with the primary data volume (probably by using the same + volume group and role), FileVault encryption could be + easily supported by the installer without requiring + manual setup by each user. + + + + - -alice$ ls -l / -lrwxr-xr-x 1 root wheel 25 Jan 1 2019 nix -> /System/Volumes/Data/nix - +
+ Change the Nix store path prefix + + Changing the default prefix for the Nix store is a simple + approach which enables you to leave it on your root volume, + where it can take full advantage of FileVault encryption if + enabled. Unfortunately, this approach also opts your device out + of some benefits that are enabled by using the same prefix + across systems: - - However builds that detect or resolve this symlink will leak the canonical - location or even fail in certain cases, making this approach undesirable. - - + + + + Your system won't be able to take advantage of the binary + cache (unless someone is able to stand up and support + duplicate caching infrastructure), which means you'll + spend more time waiting for builds. + + + + + It's harder to build and deploy packages to Linux systems. + + + + - - - An empty directory can also be created using /etc/synthetic.conf, - this won't be writable but can be used as a mount point. And with - APFS it's relatively easy to create an separate - volume for nix instead. - + - -nix - + It would also possible (and often requested) to just apply this + change ecosystem-wide, but it's an intrusive process that has + side effects we want to avoid for now. + + + + +
- -alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B -alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix -alice$ mount -/dev/disk1s6 on /nix (apfs, local, journaled) - +
+ Use a separate encrypted volume + + If you like, you can also add encryption to the recommended + approach taken by the installer. You can do this by pre-creating + an encrypted volume before you run the installer--or you can + run the installer and encrypt the volume it creates later. + + + + In either case, adding encryption to a second volume isn't quite + as simple as enabling FileVault for your boot volume. Before you + dive in, there are a few things to weigh: + + + + + The additional volume won't be encrypted with your existing + FileVault key, so you'll need another mechanism to decrypt + the volume. + + + + + You can store the password in Keychain to automatically + decrypt the volume on boot--but it'll have to wait on Keychain + and may not mount before your GUI apps restore. If any of + your launchd agents or apps depend on Nix-installed software + (for example, if you use a Nix-installed login shell), the + restore may fail or break. + + + On a case-by-case basis, you may be able to work around this + problem by using wait4path to block + execution until your executable is available. + + + It's also possible to decrypt and mount the volume earlier + with a login hook--but this mechanism appears to be + deprecated and its future is unclear. + + + + + You can hard-code the password in the clear, so that your + store volume can be decrypted before Keychain is available. + + + + + If you are comfortable navigating these tradeoffs, you can encrypt the volume with + something along the lines of: + + + + alice$ diskutil apfs enableFileVault /nix -user disk + + +
+ +
+ + Symlink the Nix store to a custom location + + Another simple approach is using /etc/synthetic.conf + to symlink the Nix store to the data volume. This option also + enables your store to share any configured FileVault encryption. + Unfortunately, builds that resolve the symlink may leak the + canonical path or even fail. + + + Because of these downsides, we can't recommend this approach. + + +
+ +
+ Notes on the recommended approach + + This section goes into a little more detail on the recommended + approach. You don't need to understand it to run the installer, + but it can serve as a helpful reference if you run into trouble. + + + + + In order to compose user-writable locations into the new + read-only system root, Apple introduced a new concept called + firmlinks, which it describes as a + "bi-directional wormhole" between two filesystems. You can + see the current firmlinks in /usr/share/firmlinks. + Unfortunately, firmlinks aren't (currently?) user-configurable. + + + + For special cases like NFS mount points or package manager roots, + synthetic.conf(5) + supports limited user-controlled file-creation (of symlinks, + and synthetic empty directories) at /. + To create a synthetic empty directory for mounting at /nix, + add the following line to /etc/synthetic.conf + (create it if necessary): + + + nix + + + + + This configuration is applied at boot time, but you can use + apfs.util to trigger creation (not deletion) + of new entries without a reboot: + + + alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B + + + + + Create the new APFS volume with diskutil: + + + alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix + + + + + Using vifs, add the new mount to + /etc/fstab. If it doesn't already have + other entries, it should look something like: + # @@ -219,29 +402,16 @@ alice$ mount # # Failure to do so is unsupported and may be destructive. # -LABEL=Nix\040Store /nix apfs rw +LABEL=Nix\040Store /nix apfs rw,nobrowse - - On macOS volumes are also mounted quite late, launchd services or other - things that start during login will start before our volume is mounted. - For these cases eg. wait4path must be used for - things that depend on /nix. - - - - This new volume also won't be encrypted by default, and enabling it - requires extra setup. For machines with a T2 chip - all data is already entrypted at rest, older hardware won't even when - FileVault is enabled for the rest of the system. - - - -alice$ diskutil apfs enableFileVault /nix -user disk - - - - + + The nobrowse setting will keep Spotlight from indexing this + volume, and keep it from showing up on your desktop. + + + +
diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index a0da85f43..47cc3e913 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -59,10 +59,45 @@ test_nix() { test -d "/nix" } -test_filevault() { +test_t2_chip_present(){ + # Use xartutil to see if system has a t2 chip. + # + # This isn't well-documented on its own; until it is, + # let's keep track of knowledge/assumptions. + # + # Warnings: + # - Don't search "xart" if porn will cause you trouble :) + # - Other xartutil flags do dangerous things. Don't run them + # naively. If you must, search "xartutil" first. + # + # Assumptions: + # - the "xART session seeds recovery utility" + # appears to interact with xartstorageremoted + # - `sudo xartutil --list` lists xART sessions + # and their seeds and exits 0 if successful. If + # not, it exits 1 and prints an error such as: + # xartutil: ERROR: No supported link to the SEP present + # - xART sessions/seeds are present when a T2 chip is + # (and not, otherwise) + # - the presence of a T2 chip means a newly-created + # volume on the primary drive will be + # encrypted at rest + # - all together: `sudo xartutil --list` + # should exit 0 if a new Nix Store volume will + # be encrypted at rest, and exit 1 if not. + sudo xartutil --list >/dev/null 2>/dev/null +} + +test_filevault_in_use() { disk=$1 - apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true || return - ! sudo xartutil --list >/dev/null 2>/dev/null + # list vols on disk | get value of Filevault key | value is true + apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true +} + +# use after error msg for conditions we don't understand +suggest_report_error(){ + # ex "error: something sad happened :(" >&2 + echo " please report this @ https://github.com/nixos/nix/issues" >&2 } main() { @@ -89,7 +124,8 @@ main() { echo "Configuring /etc/synthetic.conf..." >&2 echo nix | sudo tee /etc/synthetic.conf if ! test_synthetic_conf; then - echo "error: failed to configure synthetic.conf" >&2 + echo "error: failed to configure synthetic.conf;" >&2 + suggest_report_error exit 1 fi fi @@ -101,7 +137,8 @@ main() { sudo mkdir -p /nix 2>/dev/null || true fi if ! test_nix; then - echo "error: failed to bootstrap /nix, a reboot might be required" >&2 + echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 + suggest_report_error exit 1 fi fi @@ -111,10 +148,25 @@ main() { if [ -z "$volume" ]; then echo "Creating a Nix Store volume..." >&2 - if test_filevault "$disk"; then - echo "error: FileVault detected, refusing to create unencrypted volume" >&2 - echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" >&2 - exit 1 + if test_filevault_in_use "$disk"; then + # TODO: Not sure if it's in-scope now, but `diskutil apfs list` + # shows both filevault and encrypted at rest status, and it + # may be the more semantic way to test for this? It'll show + # `FileVault: No (Encrypted at rest)` + # `FileVault: No` + # `FileVault: Yes (Unlocked)` + # and so on. + if test_t2_chip_present; then + echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 + echo " is only encrypted at rest." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + else + echo "error: refusing to create Nix store volume because the boot volume is" >&2 + echo " FileVault encrypted, but encryption-at-rest is not available." >&2 + echo " Manually create a volume for the store and re-run this script." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + exit 1 + fi fi sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix @@ -128,13 +180,6 @@ main() { label=$(echo "$volume" | sed 's/ /\\040/g') printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs fi - - echo "" >&2 - echo "The following options can be enabled to disable spotlight indexing" >&2 - echo "of the volume, which might be desirable." >&2 - echo "" >&2 - echo " $ sudo mdutil -i off /nix" >&2 - echo "" >&2 } main "$@" diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 2f291ed4c..72aa5abf5 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -52,7 +52,7 @@ while [ $# -gt 0 ]; do NIX_INSTALLER_NO_CHANNEL_ADD=1;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; - --create-volume) + --darwin-use-unencrypted-nix-store-volume) CREATE_DARWIN_VOLUME=1;; *) ( @@ -77,12 +77,13 @@ while [ $# -gt 0 ]; do echo "" ) >&2 - if [ "$(uname -s)" = "Darwin" ]; then + # darwin and Catalina+ + if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then ( - echo " --create-volume: Create an APFS volume for the store and create the /nix" - echo " mountpoint for it using synthetic.conf." - echo " (required on macOS >=10.15)" - echo " See https://nixos.org/nix/manual/#sect-apfs-volume-installation" + echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" + echo " store and mount it at /nix. This is the recommended way to create" + echo " /nix with a read-only / on macOS >=10.15." + echo " See: https://nixos.org/nix/manual/#sect-macos-installation" echo "" ) >&2 fi @@ -98,12 +99,12 @@ if [ "$(uname -s)" = "Darwin" ]; then fi info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) - if ! [ -e $dest ] && [ -n "$info" ]; then + if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then ( echo "" echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." - echo "Use sh <(curl https://nixos.org/nix/install) --create-volume or run the preparation steps manually." - echo "See https://nixos.org/nix/manual/#sect-apfs-volume-installation" + echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-macos-installation" echo "" ) >&2 exit 1 From d3df1889a1f49b114a8d78270ea8a6d0523f5e35 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 21 May 2020 20:03:09 +0200 Subject: [PATCH 271/350] installer: don't clobber synthetic.conf --- scripts/create-darwin-volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 47cc3e913..5214d1652 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -122,7 +122,7 @@ main() { if ! test_synthetic_conf; then echo "Configuring /etc/synthetic.conf..." >&2 - echo nix | sudo tee /etc/synthetic.conf + echo nix | sudo tee -a /etc/synthetic.conf if ! test_synthetic_conf; then echo "error: failed to configure synthetic.conf;" >&2 suggest_report_error From 909d8cb2934869c945ac1cc20dfb71df513042eb Mon Sep 17 00:00:00 2001 From: Suraj Barkale Date: Fri, 22 May 2020 11:05:25 +1000 Subject: [PATCH 272/350] Use /etc/zshenv instead of /etc/zshrc for profile As noted in https://github.com/NixOS/nix/issues/3456 the `/etc/zshenv` file provides a better place for sourcing the nix environment. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a0f1deb98..838192733 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -28,7 +28,7 @@ readonly NIX_FIRST_BUILD_UID="30001" # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" From 2a7ea2eb6c54c82d5e858ea6ae9de929face5e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Sat, 23 May 2020 11:12:05 +0200 Subject: [PATCH 273/350] scripts/create-darwin-volume.sh: remove unused variable --- scripts/create-darwin-volume.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 5214d1652..dac30d72d 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -15,7 +15,7 @@ disk_identifier() { } volume_list_true() { - key=$1 t=$2 + key=$1 xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null } From 6f6bdd63a0f6dae4ab91422645f548837029dee9 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 23 May 2020 12:10:12 +0200 Subject: [PATCH 274/350] fix hydra build products Since the binary tarball was replaced none of the hydra builds include the manual. The dist phase isn't enabled by default the manual build products where not written. --- release.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release.nix b/release.nix index 2cc4bb4c0..2a320e1c3 100644 --- a/release.nix +++ b/release.nix @@ -115,17 +115,17 @@ let installFlags = "sysconfdir=$(out)/etc"; + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ''; + doCheck = true; doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; separateDebugInfo = true; - - preDist = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ''; }); From e2af11ce071fc109ad3e517bf15a87024d0a9ae5 Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Sat, 23 May 2020 15:26:59 +0200 Subject: [PATCH 275/350] Manpages: Do not refer to nixpkgs-channels Unless I am misinformed, using the `nixpkgs` repository directly is now preferred? --- doc/manual/command-ref/env-common.xml | 2 +- doc/manual/command-ref/nix-env.xml | 7 ++----- doc/manual/command-ref/nix-shell.xml | 6 +++--- doc/manual/expressions/builtins.xml | 4 ++-- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 0217de7b2..8466cc463 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -53,7 +53,7 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos NIX_PATH to -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz +nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09 channel. diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 9c03ccce1..2b95b6819 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): 14.12 channel: -$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox -(The GitHub repository nixpkgs-channels is updated -automatically from the main nixpkgs repository -after certain tests have succeeded and binaries have been built and -uploaded to the binary cache at cache.nixos.org.) + diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 766482460..2fef323c5 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -258,7 +258,7 @@ path. You can override it by passing or setting containing the Pan package from a specific revision of Nixpkgs: -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 6709c4081..a18c5801a 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" particular version of Nixpkgs, e.g. -with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; +with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; stdenv.mkDerivation { … } @@ -349,7 +349,7 @@ stdenv.mkDerivation { … } with import (fetchTarball { - url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz"; + url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz"; sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; }) {}; From ecc5c90dfc0e9877aef89bf0192203cd1a8f2b21 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Mon, 25 May 2020 11:50:41 +0200 Subject: [PATCH 276/350] Add unit tests for hashing functions --- src/libutil/tests/hash.cc | 80 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/libutil/tests/hash.cc diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc new file mode 100644 index 000000000..7cb439817 --- /dev/null +++ b/src/libutil/tests/hash.cc @@ -0,0 +1,80 @@ +#include "hash.hh" +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * hashString + * --------------------------------------------------------------------------*/ + + TEST(hashString, testKnownMD5Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s1 = ""; + auto hash = hashString(HashType::htMD5, s1); + ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e"); + } + + TEST(hashString, testKnownMD5Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s2 = "abc"; + auto hash = hashString(HashType::htMD5, s2); + ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72"); + } + + TEST(hashString, testKnownSHA1Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abc"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + } + + TEST(hashString, testKnownSHA1Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + } + + TEST(hashString, testKnownSHA256Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + TEST(hashString, testKnownSHA256Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + } + + TEST(hashString, testKnownSHA512Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" + "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" + "454d4423643ce80e2a9ac94fa54ca49f"); + } + + TEST(hashString, testKnownSHA512Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16), + "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" + "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" + "c7d329eeb6dd26545e96e55b874be909"); + } + + TEST(hashString, hashingWithUnknownAlgoExits) { + auto s = "unknown"; + ASSERT_DEATH(hashString(HashType::htUnknown, s), ""); + } +} From c284700867ce09abee1f891a402271a42d207b4d Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Mon, 25 May 2020 11:57:45 +0200 Subject: [PATCH 277/350] Add unit tests for "json.hh" --- src/libutil/tests/json.cc | 193 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 src/libutil/tests/json.cc diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc new file mode 100644 index 000000000..dea73f53a --- /dev/null +++ b/src/libutil/tests/json.cc @@ -0,0 +1,193 @@ +#include "json.hh" +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * toJSON + * --------------------------------------------------------------------------*/ + + TEST(toJSON, quotesCharPtr) { + const char* input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, quotesStdString) { + std::string input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, convertsNullptrtoNull) { + auto input = nullptr; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + TEST(toJSON, convertsNullToNull) { + const char* input = 0; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + + TEST(toJSON, convertsFloat) { + auto input = 1.024f; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsDouble) { + const double input = 1.024; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsBool) { + auto input = false; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "false"); + } + + TEST(toJSON, quotesTab) { + std::stringstream out; + toJSON(out, "\t"); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + TEST(toJSON, quotesNewline) { + std::stringstream out; + toJSON(out, "\n"); + + ASSERT_EQ(out.str(), "\"\\n\""); + } + + TEST(toJSON, quotesCreturn) { + std::stringstream out; + toJSON(out, "\r"); + + ASSERT_EQ(out.str(), "\"\\r\""); + } + + TEST(toJSON, quotesCreturnNewLine) { + std::stringstream out; + toJSON(out, "\r\n"); + + ASSERT_EQ(out.str(), "\"\\r\\n\""); + } + + TEST(toJSON, quotesDoublequotes) { + std::stringstream out; + toJSON(out, "\""); + + ASSERT_EQ(out.str(), "\"\\\"\""); + } + + TEST(toJSON, substringEscape) { + std::stringstream out; + const char *s = "foo\t"; + toJSON(out, s+3, s + strlen(s)); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + /* ---------------------------------------------------------------------------- + * JSONObject + * --------------------------------------------------------------------------*/ + + TEST(JSONObject, emptyObject) { + std::stringstream out; + { + JSONObject t(out); + } + ASSERT_EQ(out.str(), "{}"); + } + + TEST(JSONObject, objectWithList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithListIndent) { + std::stringstream out; + { + JSONObject t(out, true); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), +R"#({ + "list": [ + "element" + ] +})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("list"); + l.list().elem("element"); + } + + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndObject) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("object"); + l.object().attr("key", "value"); + } + + ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); + } + + /* ---------------------------------------------------------------------------- + * JSONList + * --------------------------------------------------------------------------*/ + + TEST(JSONList, empty) { + std::stringstream out; + { + JSONList l(out); + } + ASSERT_EQ(out.str(), R"#([])#"); + } + + TEST(JSONList, withElements) { + std::stringstream out; + { + JSONList l(out); + l.elem("one"); + l.object(); + l.placeholder().write("three"); + } + ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); + } +} + From 90b0c630a0e9e66f69f1d24b538982c20b5486b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Wed, 26 Feb 2020 10:50:40 +0100 Subject: [PATCH 278/350] install-multi-user: allow overriding user count --- scripts/install-multi-user.sh | 4 +++- scripts/install-nix-from-closure.sh | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a0f1deb98..991cf1998 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -20,7 +20,9 @@ readonly GREEN='\033[32m' readonly GREEN_UL='\033[4;32m' readonly RED='\033[31m' -readonly NIX_USER_COUNT="32" +# installer allows overriding build user count to speed up installation +# as creating each user takes non-trivial amount of time on macos +readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" readonly NIX_FIRST_BUILD_UID="30001" diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 72aa5abf5..6ea23a44a 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -50,6 +50,9 @@ while [ $# -gt 0 ]; do INSTALL_MODE=no-daemon;; --no-channel-add) NIX_INSTALLER_NO_CHANNEL_ADD=1;; + --daemon-user-count) + NIX_USER_COUNT=$2 + shift;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) From 573ff8dfcaacc4495e7f7880d86f70145a074578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Mon, 25 May 2020 17:31:46 +0200 Subject: [PATCH 279/350] Allow passing extra nix.conf to installer --- scripts/install-multi-user.sh | 1 + scripts/install-nix-from-closure.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a0f1deb98..b0cb51943 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -646,6 +646,7 @@ setup_default_profile() { place_nix_configuration() { cat < "$SCRATCH/nix.conf" +$NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 72aa5abf5..3e0312c78 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -41,6 +41,7 @@ fi INSTALL_MODE=no-daemon CREATE_DARWIN_VOLUME=0 +NIX_EXTRA_CONF= # handle the command line flags while [ $# -gt 0 ]; do case $1 in @@ -54,6 +55,9 @@ while [ $# -gt 0 ]; do NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) CREATE_DARWIN_VOLUME=1;; + --nix-extra-conf-file) + NIX_EXTRA_CONF=$(cat $2) + shift;; *) ( echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]" From 4b388e84318722f970326967e1648675655740fe Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Mon, 25 May 2020 18:34:55 +0200 Subject: [PATCH 280/350] Add unit tests for xml-writer --- src/libutil/tests/xml-writer.cc | 105 ++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 src/libutil/tests/xml-writer.cc diff --git a/src/libutil/tests/xml-writer.cc b/src/libutil/tests/xml-writer.cc new file mode 100644 index 000000000..adcde25c9 --- /dev/null +++ b/src/libutil/tests/xml-writer.cc @@ -0,0 +1,105 @@ +#include "xml-writer.hh" +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * XMLWriter + * --------------------------------------------------------------------------*/ + + TEST(XMLWriter, emptyObject) { + std::stringstream out; + { + XMLWriter t(false, out); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.openElement("foobar"); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithEmptyAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = {}; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, objectWithElementWithAttrsEscaping) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "", "" } + }; + t.openElement("foobar", attrs); + } + + // XXX: While "" is escaped, "" isn't which I think is a bug. + ASSERT_EQ(out.str(), "\n=\"<value>\">"); + } + + TEST(XMLWriter, objectWithElementWithAttrsIndented) { + std::stringstream out; + { + XMLWriter t(true, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "\n\n\n"); + } + + TEST(XMLWriter, writeEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.writeEmptyElement("foobar"); + } + + ASSERT_EQ(out.str(), "\n"); + } + + TEST(XMLWriter, writeEmptyElementWithAttributes) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.writeEmptyElement("foobar", attrs); + + } + + ASSERT_EQ(out.str(), "\n"); + } + +} From 1a5ac894e929cefceec22c0f1ca248ee6be445ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 26 May 2020 15:49:26 +0200 Subject: [PATCH 281/350] Fix installer script bugs - --no-channel-add didn't have effect on multi-user installation - some new flags didn't work at all - document all installer flags --- scripts/install-multi-user.sh | 24 ++++++++++++++---------- scripts/install-nix-from-closure.sh | 13 ++++++++----- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 74cc6a5b0..ab41e0242 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -29,6 +29,7 @@ readonly NIX_FIRST_BUILD_UID="30001" # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" +readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" @@ -452,9 +453,11 @@ create_directories() { } place_channel_configuration() { - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" - _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + fi } welcome_to_nix() { @@ -636,13 +639,14 @@ setup_default_profile() { export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt fi - # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, - # otherwise it will be lost in environments where sudo doesn't pass - # all the environment variables by default. - _sudo "to update the default channel in the default profile" \ - HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ - || channel_update_failed=1 - + if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, + # otherwise it will be lost in environments where sudo doesn't pass + # all the environment variables by default. + _sudo "to update the default channel in the default profile" \ + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ + || channel_update_failed=1 + fi } diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 635aaa16d..826ca8b8c 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -41,7 +41,6 @@ fi INSTALL_MODE=no-daemon CREATE_DARWIN_VOLUME=0 -NIX_EXTRA_CONF= # handle the command line flags while [ $# -gt 0 ]; do case $1 in @@ -50,20 +49,20 @@ while [ $# -gt 0 ]; do --no-daemon) INSTALL_MODE=no-daemon;; --no-channel-add) - NIX_INSTALLER_NO_CHANNEL_ADD=1;; + export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) - NIX_USER_COUNT=$2 + export NIX_USER_COUNT=$2 shift;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; --darwin-use-unencrypted-nix-store-volume) CREATE_DARWIN_VOLUME=1;; --nix-extra-conf-file) - NIX_EXTRA_CONF=$(cat $2) + export NIX_EXTRA_CONF="$(cat $2)" shift;; *) ( - echo "Nix Installer [--daemon|--no-daemon] [--no-channel-add] [--no-modify-profile]" + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" echo "Choose installation method." echo "" @@ -82,6 +81,10 @@ while [ $# -gt 0 ]; do echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" echo " is installed by default." echo "" + echo " --daemon-user-count: Number of build users to create. Defaults to 32." + echo "" + echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" + echo "" ) >&2 # darwin and Catalina+ From 3d3c219d917525b0a131c4332dd65eadfc818f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Tue, 26 May 2020 16:23:03 +0200 Subject: [PATCH 282/350] installer: fix unused variable --- scripts/install-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ab41e0242..d04b4bbbf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -453,7 +453,7 @@ create_directories() { } place_channel_configuration() { - if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" _sudo "to set up the default system channel (part 1)" \ install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" @@ -639,7 +639,7 @@ setup_default_profile() { export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt fi - if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, # otherwise it will be lost in environments where sudo doesn't pass # all the environment variables by default. From b90241ceb13500238e428f8943d221f24bf5322b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Tue, 26 May 2020 11:32:41 -0400 Subject: [PATCH 283/350] Change remaining bools with FileIngestionMethod --- src/libfetchers/git.cc | 4 ++-- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 6 +++--- src/libfetchers/tree-info.cc | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7c18cf67f..17cc60228 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -195,7 +195,7 @@ struct GitInput : Input return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); auto tree = Tree { .actualPath = store->printStorePath(storePath), @@ -347,7 +347,7 @@ struct GitInput : Input unpackTarfile(*source, tmpDir); } - auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 1d6571571..2e0d4bf4d 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -114,7 +114,7 @@ struct MercurialInput : Input return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return {Tree { .actualPath = store->printStorePath(storePath), diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..bf2b2a5ff 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -67,10 +67,10 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); store->addToStore(info, sink.s, NoRepair, NoCheckSigs); storePath = std::move(info.path); } @@ -141,7 +141,7 @@ Tree downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 5788e94a1..b2d8cfc8d 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,7 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); } } From 93129cf1ddf7f91ddec11ba7a6a364ab54f345c5 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Tue, 26 May 2020 17:49:31 +0200 Subject: [PATCH 284/350] Add unit tests for config.cc --- src/libutil/tests/config.cc | 204 ++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 src/libutil/tests/config.cc diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc new file mode 100644 index 000000000..b0be24d5e --- /dev/null +++ b/src/libutil/tests/config.cc @@ -0,0 +1,204 @@ +#include "json.hh" +#include "config.hh" +#include "args.hh" + +#include +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * Config + * --------------------------------------------------------------------------*/ + + TEST(Config, setUndefinedSetting) { + Config config; + ASSERT_EQ(config.set("undefined-key", "value"), false); + } + + TEST(Config, setDefinedSetting) { + Config config; + std::string value; + Setting foo{&config, value, "name-of-the-setting", "description"}; + ASSERT_EQ(config.set("name-of-the-setting", "value"), true); + } + + TEST(Config, getDefinedSetting) { + Config config; + std::string value; + std::map settings; + Setting foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, ""); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedOverridenSettingNotSet) { + Config config; + std::string value; + std::map settings; + Setting foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ true); + const auto e = settings.find("name-of-the-setting"); + ASSERT_EQ(e, settings.end()); + } + + TEST(Config, getDefinedSettingSet1) { + Config config; + std::string value; + std::map settings; + Setting setting{&config, value, "name-of-the-setting", "description"}; + + setting.assign("value"); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, "value"); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedSettingSet2) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_TRUE(config.set("name-of-the-setting", "value")); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto e = settings.find("name-of-the-setting"); + ASSERT_NE(e, settings.end()); + ASSERT_EQ(e->second.value, "value"); + ASSERT_EQ(e->second.description, "description"); + } + + TEST(Config, addSetting) { + class TestSetting : public AbstractSetting { + public: + TestSetting() : AbstractSetting("test", "test", {}) {} + void set(const std::string & value) {} + std::string to_string() const { return {}; } + }; + + Config config; + TestSetting setting; + + ASSERT_FALSE(config.set("test", "value")); + config.addSetting(&setting); + ASSERT_TRUE(config.set("test", "value")); + } + + TEST(Config, withInitialValue) { + const StringMap initials = { + { "key", "value" }, + }; + Config config(initials); + + { + std::map settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings.find("key"), settings.end()); + } + + Setting setting{&config, "default-value", "key", "description"}; + + { + std::map settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings["key"].value, "value"); + } + } + + TEST(Config, resetOverriden) { + Config config; + config.resetOverriden(); + } + + TEST(Config, resetOverridenWithSetting) { + Config config; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + { + std::map settings; + + setting.set("foo"); + ASSERT_EQ(setting.get(), "foo"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + + { + std::map settings; + + setting.override("bar"); + ASSERT_TRUE(setting.overriden); + ASSERT_EQ(setting.get(), "bar"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_FALSE(settings.empty()); + } + + { + std::map settings; + + config.resetOverriden(); + ASSERT_FALSE(setting.overriden); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + } + + TEST(Config, toJSONOnEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + Config config; + config.toJSON(obj); + } + + ASSERT_EQ(out.str(), "{}"); + } + + TEST(Config, toJSONOnNonEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + setting.assign("value"); + + config.toJSON(obj); + } + ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#"); + } + + TEST(Config, setSettingAlias) { + Config config; + Setting setting{&config, "", "some-int", "best number", { "another-int" }}; + ASSERT_TRUE(config.set("some-int", "1")); + ASSERT_EQ(setting.get(), "1"); + ASSERT_TRUE(config.set("another-int", "2")); + ASSERT_EQ(setting.get(), "2"); + ASSERT_TRUE(config.set("some-int", "3")); + ASSERT_EQ(setting.get(), "3"); + } + + /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything + * useful (these days). Whenever we add a new setting to Config the + * unknown settings are always considered. In which case is this function + * actually useful? Is there some way to register a Setting without calling + * addSetting? */ + TEST(Config, DISABLED_reapplyUnknownSettings) { + Config config; + ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue")); + Setting setting{&config, "default", "name-of-the-setting", "description"}; + ASSERT_EQ(setting.get(), "default"); + config.reapplyUnknownSettings(); + ASSERT_EQ(setting.get(), "unknownvalue"); + } +} From e1b8c64c04b40694418de87e3e0d118cf0677bfc Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 27 May 2020 14:39:02 +0200 Subject: [PATCH 285/350] config.cc: extract parts of applyConfigFile into applyConfig This moves the actual parsing of configuration contents into applyConfig which applyConfigFile is then going to call. By changing this we can now test the configuration file parsing without actually create a file on disk. --- src/libutil/config.cc | 103 ++++++++++++++++++++++-------------------- src/libutil/config.hh | 1 + 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index f03e444ec..8fc700a2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -65,60 +65,63 @@ void Config::getSettings(std::map & res, bool override res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector tokens = tokenizeString >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + string name = tokens[0]; + + vector::iterator i = tokens.begin(); + advance(i, 2); + + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; +} + void AbstractConfig::applyConfigFile(const Path & path) { try { string contents = readFile(path); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector tokens = tokenizeString >(line); - if (tokens.empty()) continue; - - if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - auto include = false; - auto ignoreMissing = false; - if (tokens[0] == "include") - include = true; - else if (tokens[0] == "!include") { - include = true; - ignoreMissing = true; - } - - if (include) { - if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); - if (pathExists(p)) { - applyConfigFile(p); - } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); - } - continue; - } - - if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - string name = tokens[0]; - - vector::iterator i = tokens.begin(); - advance(i, 2); - - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - }; + applyConfig(contents, path); } catch (SysError &) { } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 7ea78fdaf..b04cba88b 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -33,6 +33,7 @@ public: virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + void applyConfig(const std::string & contents, const std::string & path = ""); void applyConfigFile(const Path & path); virtual void resetOverriden() = 0; From 9df3d8ccd7a3130faccb73933d0c196a81d605eb Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 27 May 2020 14:41:01 +0200 Subject: [PATCH 286/350] tests/config.cc: add tests for Config::applyConfig --- src/libutil/tests/config.cc | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index b0be24d5e..74c59fd31 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -201,4 +201,64 @@ namespace nix { config.reapplyUnknownSettings(); ASSERT_EQ(setting.get(), "unknownvalue"); } + + TEST(Config, applyConfigEmpty) { + Config config; + std::map settings; + config.applyConfig(""); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigEmptyWithComment) { + Config config; + std::map settings; + config.applyConfig("# just a comment"); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigAssignment) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = value-from-file #useful comment\n" + "# name-of-the-setting = foo\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file"); + } + + TEST(Config, applyConfigWithReassignedSetting) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = first-value\n" + "name-of-the-setting = second-value\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "second-value"); + } + + TEST(Config, applyConfigFailsOnMissingIncludes) { + Config config; + std::map settings; + Setting setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_THROW(config.applyConfig( + "name-of-the-setting = value-from-file\n" + "# name-of-the-setting = foo\n" + "include /nix/store/does/not/exist.nix" + ), Error); + } + + TEST(Config, applyConfigInvalidThrows) { + Config config; + ASSERT_THROW(config.applyConfig("value == key"), UsageError); + ASSERT_THROW(config.applyConfig("value "), UsageError); + } } From fc137d2f007c32f27a49e1a00d5114c7d7663419 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 27 May 2020 16:37:24 +0200 Subject: [PATCH 287/350] config.hh: Add documentation Provides some general overview on the mechanics of Config/Setting and comments for the public methods of Config. --- src/libutil/config.hh | 69 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index b04cba88b..5c7a70a2e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -7,6 +7,38 @@ namespace nix { +/** + * The Config class provides Nix runtime configurations. + * + * What is a Configuration? + * A collection of uniquely named Settings. + * + * What is a Setting? + * Each property that you can set in a configuration corresponds to a + * `Setting`. A setting records value and description of a property + * with a default and optional aliases. + * + * A valid configuration consists of settings that are registered to a + * `Config` object instance: + * + * Config config; + * Setting systemSetting{&config, "x86_64-linux", "system", "the current system"}; + * + * The above creates a `Config` object and registers a setting called "system" + * via the variable `systemSetting` with it. The setting defaults to the string + * "x86_64-linux", it's description is "the current system". All of the + * registered settings can then be accessed as shown below: + * + * std::map settings; + * config.getSettings(settings); + * config["system"].description == "the current system" + * config["system"].value == "x86_64-linux" + * + * + * The above retrieves all currently known settings from the `Config` object + * and adds them to the `settings` map. + */ + class Args; class AbstractSetting; class JSONPlaceholder; @@ -23,6 +55,10 @@ protected: public: + /** + * Sets the value referenced by `name` to `value`. Returns true if the + * setting is known, false otherwise. + */ virtual bool set(const std::string & name, const std::string & value) = 0; struct SettingInfo @@ -31,19 +67,52 @@ public: std::string description; }; + /** + * Adds the currently known settings to the given result map `res`. + * - res: map to store settings in + * - overridenOnly: when set to true only overridden settings will be added to `res` + */ virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + /** + * Parses the configuration in `contents` and applies it + * - contents: configuration contents to be parsed and applied + * - path: location of the configuration file + */ void applyConfig(const std::string & contents, const std::string & path = ""); + + /** + * Applies a nix configuration file + * - path: the location of the config file to apply + */ void applyConfigFile(const Path & path); + /** + * Resets the `overridden` flag of all Settings + */ virtual void resetOverriden() = 0; + /** + * Outputs all settings to JSON + * - out: JSONObject to write the configuration to + */ virtual void toJSON(JSONObject & out) = 0; + /** + * Converts settings to `Args` to be used on the command line interface + * - args: args to write to + * - category: category of the settings + */ virtual void convertToArgs(Args & args, const std::string & category) = 0; + /** + * Logs a warning for each unregistered setting + */ void warnUnknownSettings(); + /** + * Re-applies all previously attempted changes to unknown settings + */ void reapplyUnknownSettings(); }; From 7873fd175d9e4e5bdb281e005b18c388bcfe1dd8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:11:45 -0500 Subject: [PATCH 288/350] =?UTF-8?q?Don=E2=80=99t=20use=20FileIngestionMeth?= =?UTF-8?q?od=20for=20StorePathsCommand?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a different recursive than used in makeFixedOutputPath. --- src/nix/command.cc | 10 +++++----- src/nix/command.hh | 4 ++-- src/nix/copy.cc | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index ea0ade88e..71b027719 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -31,21 +31,21 @@ void StoreCommand::run() run(getStore()); } -StorePathsCommand::StorePathsCommand(FileIngestionMethod recursive) +StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { - if (recursive == FileIngestionMethod::Recursive) + if (recursive) addFlag({ .longName = "no-recursive", .description = "apply operation to specified paths only", - .handler = {&this->recursive, FileIngestionMethod::Flat}, + .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', .description = "apply operation to closure of the specified paths", - .handler = {&this->recursive, FileIngestionMethod::Recursive}, + .handler = {&this->recursive, true}, }); mkFlag(0, "all", "apply operation to the entire store", &all); @@ -66,7 +66,7 @@ void StorePathsCommand::run(ref store) for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p.clone()); - if (recursive == FileIngestionMethod::Recursive) { + if (recursive) { StorePathSet closure; store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); storePaths.clear(); diff --git a/src/nix/command.hh b/src/nix/command.hh index 09c621b5b..959d5f19d 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -92,7 +92,7 @@ struct StorePathsCommand : public InstallablesCommand { private: - FileIngestionMethod recursive = FileIngestionMethod::Flat; + bool recursive = false; bool all = false; protected: @@ -101,7 +101,7 @@ protected: public: - StorePathsCommand(FileIngestionMethod recursive = FileIngestionMethod::Flat); + StorePathsCommand(bool recursive = false); using StoreCommand::run; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 60aa3f14b..c7c38709d 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -17,7 +17,7 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; CmdCopy() - : StorePathsCommand(FileIngestionMethod::Recursive) + : StorePathsCommand(true) { addFlag({ .longName = "from", From c66441a646c2bbfab05fa4a4fd05c2ae897dda8b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 27 May 2020 13:04:20 -0500 Subject: [PATCH 289/350] =?UTF-8?q?Rename=20some=20variables=20named=20?= =?UTF-8?q?=E2=80=9Crecursive=E2=80=9D=20to=20=E2=80=9Cmethod=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is much less confusing since recursive is no longer a boolean. --- src/libexpr/primops.cc | 14 +++++++------- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build.cc | 18 +++++++++--------- src/libstore/local-store.cc | 22 +++++++++++----------- src/libstore/store-api.cc | 6 +++--- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fcbb75ceb..d458ab272 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1042,7 +1042,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod recursive, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1073,12 +1073,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -1108,7 +1108,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Path path; string name; Value * filterFun = nullptr; - auto recursive = FileIngestionMethod::Recursive; + auto method = FileIngestionMethod::Recursive; Hash expectedHash; for (auto & attr : *args[0]->attrs) { @@ -1124,7 +1124,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") - recursive = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; + method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; else if (n == "sha256") expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else @@ -1135,7 +1135,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a61af4a00..f13736c58 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -327,7 +327,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +336,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -345,7 +345,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); addToStore(info, sink.s, repair, CheckSigs, nullptr); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 1a1ea636c..4ff5609ed 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -79,7 +79,7 @@ public: std::shared_ptr accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 272c4f1ae..f5c132a83 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2734,7 +2734,7 @@ struct RestrictedStore : public LocalFSStore { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2747,9 +2747,9 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod recursive = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { - auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); return path; } @@ -3692,10 +3692,10 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - FileIngestionMethod recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (recursive == FileIngestionMethod::Flat) { + if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3705,11 +3705,11 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive == FileIngestionMethod::Recursive + Hash h2 = outputHashMode == FileIngestionMethod::Recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); if (h != h2) { @@ -3738,7 +3738,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 746f81beb..80851b591 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -557,10 +557,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - FileIngestionMethod recursive; Hash h; - out->second.parseHashInfo(recursive, h); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { @@ -1043,11 +1043,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1067,7 +1067,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1080,7 +1080,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive == FileIngestionMethod::Recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1103,7 +1103,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1111,12 +1111,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive == FileIngestionMethod::Recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 86cb20a26..d479b86cb 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -204,12 +204,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive == FileIngestionMethod::Recursive + Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(recursive, h, name), h); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } From 4e6d7cb55a884ac19990b487c2020ea257e0f9c3 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Wed, 27 May 2020 20:58:42 +0200 Subject: [PATCH 290/350] installer: don't require xz on darwin On macOS the system tar has builtin support for lzma while xz isn't available as a separate binary. There's no builtin package manager there available either so having to install lzma (without nix) would be rather painful. --- scripts/install.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/install.in b/scripts/install.in index 6709f00d4..1d26c4ff0 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -36,7 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" -require_util xz "unpack the binary tarball" +if [ "$(uname -s)" != "Darwin" ]; then + require_util xz "unpack the binary tarball" +fi echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" From 0f96f45061e574c9cf217e3dd5634dfdbf210235 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 27 May 2020 23:50:11 -0400 Subject: [PATCH 291/350] Use `FileIngestionMethod` for `nix hash` There was an enum there that matched in perfectly. --- src/nix/hash.cc | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 366314227..0a7c343d0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,15 +9,14 @@ using namespace nix; struct CmdHash : Command { - enum Mode { mFile, mPath }; - Mode mode; + FileIngestionMethod mode; Base base = SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; std::optional modulus; - CmdHash(Mode mode) : mode(mode) + CmdHash(FileIngestionMethod mode) : mode(mode) { mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); @@ -36,9 +35,14 @@ struct CmdHash : Command std::string description() override { - return mode == mFile - ? "print cryptographic hash of a regular file" - : "print cryptographic hash of the NAR serialisation of a path"; + const char* d; + switch (mode) { + case FileIngestionMethod::Flat: + d = "print cryptographic hash of a regular file"; + case FileIngestionMethod::Recursive: + d = "print cryptographic hash of the NAR serialisation of a path"; + }; + return d; } Category category() override { return catUtility; } @@ -53,10 +57,14 @@ struct CmdHash : Command else hashSink = std::make_unique(ht); - if (mode == mFile) + switch (mode) { + case FileIngestionMethod::Flat: readFile(path, *hashSink); - else + break; + case FileIngestionMethod::Recursive: dumpPath(path, *hashSink); + break; + } Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); @@ -65,8 +73,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1("hash-file", [](){ return make_ref(CmdHash::mFile); }); -static RegisterCommand r2("hash-path", [](){ return make_ref(CmdHash::mPath); }); +static RegisterCommand r1("hash-file", [](){ return make_ref(FileIngestionMethod::Flat); }); +static RegisterCommand r2("hash-path", [](){ return make_ref(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -137,7 +145,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; From fac0c2d54a6b04175b40009506f2720d2594ed4e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 29 May 2020 16:19:48 -0400 Subject: [PATCH 292/350] Remove addToStore variant as requested by `FIXME` The idea is it's always more flexible to consumer a `Source` than a plain string, and it might even reduce memory consumption. I also looked at `addToStoreFromDump` with its `// FIXME: remove?`, but the worked needed for that is far more up for interpretation, so I punted for now. --- src/libfetchers/tarball.cc | 3 ++- src/libstore/binary-cache-store.cc | 11 ++++++++--- src/libstore/binary-cache-store.hh | 2 +- src/libstore/export-import.cc | 5 ++++- src/libstore/store-api.cc | 15 --------------- src/libstore/store-api.hh | 7 +------ src/nix/add-to-store.cc | 6 ++++-- 7 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index bf2b2a5ff..b6e57379b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,7 +71,8 @@ DownloadFileResult downloadFile( info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + auto source = StringSource { *sink.s }; + store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f13736c58..357962007 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref & nar, +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) { + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref(narSource.drain()); + if (!repair && isValidPath(info.path)) return; /* Verify that all references are valid. This may do some .narinfo @@ -347,7 +350,8 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath ValidPathInfo info(makeFixedOutputPath(method, h, name)); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); return std::move(info.path); } @@ -361,7 +365,8 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); } return std::move(info.path); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 4ff5609ed..52ef8aa7a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -74,7 +74,7 @@ public: std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - void addToStore(const ValidPathInfo & info, const ref & nar, + void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) override; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4692d1a7b..f0d01a240 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,3 +1,4 @@ +#include "serialise.hh" #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" @@ -100,7 +101,9 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces if (readInt(source) == 1) readString(source); - addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + // Can't use underlying source, which would have been exhausted + auto source = StringSource { *tee.source.data }; + addToStore(info, source, NoRepair, checkSigs, accessor); res.push_back(info.path.clone()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d479b86cb..095363d0c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -840,21 +840,6 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) } -void Store::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) -{ - addToStore(info, make_ref(narSource.drain()), repair, checkSigs, accessor); -} - -void Store::addToStore(const ValidPathInfo & info, const ref & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr accessor) -{ - StringSource source(*nar); - addToStore(info, source, repair, checkSigs, accessor); -} - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3def209a6..b1e25fc7d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -450,12 +450,7 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0); - - // FIXME: remove - virtual void addToStore(const ValidPathInfo & info, const ref & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr accessor = 0); + std::shared_ptr accessor = 0) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 4b4ba81cb..f43f774c1 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,8 +50,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); - if (!dryRun) - store->addToStore(info, sink.s); + if (!dryRun) { + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + } logger->stdout("%s", store->printStorePath(info.path)); } From 77007d4eabcf7090d1d3fbbdf84a67fb2262cf78 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Sat, 30 May 2020 12:29:35 +0200 Subject: [PATCH 293/350] Improve ref validity checking in fetchGit The previous regex was too strict and did not match what git was allowing. It could lead to `fetchGit` not accepting valid branch names, even though they exist in a repository (for example, branch names containing `/`, which are pretty standard, like `release/1.0` branches). The new regex defines what a branch name should **NOT** contain. It takes the definitions from `refs.c` in https://github.com/git/git and `git help check-ref-format` pages. This change also introduces a test for ref name validity checking, which compares the result from Nix with the result of `git check-ref-format --branch`. --- src/libfetchers/git.cc | 2 +- src/libutil/url.cc | 1 + src/libutil/url.hh | 6 +++ tests/fetchGitRefs.sh | 111 +++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + 5 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 tests/fetchGitRefs.sh diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 17cc60228..e069057eb 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -418,7 +418,7 @@ struct GitInputScheme : InputScheme auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) + if (std::regex_search(*ref, badGitRefRegex)) throw BadURL("invalid Git branch/tag name '%s'", *ref); input->ref = *ref; } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5d5328e5d..88c09eef9 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -4,6 +4,7 @@ namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 1503023a2..4a0d4071b 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -49,6 +49,12 @@ const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRege const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check extern std::regex refRegex; +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + // A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh new file mode 100644 index 000000000..93993ae90 --- /dev/null +++ b/tests/fetchGitRefs.sh @@ -0,0 +1,111 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git tests" + exit 99 +fi + +clearStore + +repo="$TEST_ROOT/git" + +rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" + +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo"/hello +git -C "$repo" add hello +git -C "$repo" commit -m 'Bla1' + +path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") + +# Test various combinations of ref names +# (taken from the git project) + +# git help check-ref-format +# Git imposes the following rules on how references are named: +# +# 1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot . or end with the sequence .lock. +# 2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived. +# 3. They cannot have two consecutive dots .. anywhere. +# 4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere. +# 5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule. +# 6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule) +# 7. They cannot end with a dot .. +# 8. They cannot contain a sequence @{. +# 9. They cannot be the single character @. +# 10. They cannot contain a \. + +valid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + git check-ref-format --branch "$1" >/dev/null + git -C "$repo" branch "$1" master >/dev/null + path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") + [[ $path1 = $path ]] + git -C "$repo" branch -D "$1" >/dev/null +} + +invalid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + # special case for a sole @: + # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel + if [ "$1" = "@" ]; then + (! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1) + else + (! git check-ref-format --branch "$1" >/dev/null 2>&1) + fi + nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'error: invalid Git branch/tag name' >/dev/null +} + + +valid_ref 'foox' +valid_ref '1337' +valid_ref 'foo.baz' +valid_ref 'foo/bar/baz' +valid_ref 'foo./bar' +valid_ref 'heads/foo@bar' +valid_ref "$(printf 'heads/fu\303\237')" +valid_ref 'foo-bar-baz' +valid_ref '$1' +valid_ref 'foo.locke' + +invalid_ref 'refs///heads/foo' +invalid_ref 'heads/foo/' +invalid_ref '///heads/foo' +invalid_ref '.foo' +invalid_ref './foo' +invalid_ref './foo/bar' +invalid_ref 'foo/./bar' +invalid_ref 'foo/bar/.' +invalid_ref 'foo bar' +invalid_ref 'foo?bar' +invalid_ref 'foo^bar' +invalid_ref 'foo~bar' +invalid_ref 'foo:bar' +invalid_ref 'foo[bar' +invalid_ref 'foo/bar/.' +invalid_ref '.refs/foo' +invalid_ref 'refs/heads/foo.' +invalid_ref 'heads/foo..bar' +invalid_ref 'heads/foo?bar' +invalid_ref 'heads/foo.lock' +invalid_ref 'heads///foo.lock' +invalid_ref 'foo.lock/bar' +invalid_ref 'foo.lock///bar' +invalid_ref 'heads/v@{ation' +invalid_ref 'heads/foo\.ar' # should fail due to \ +invalid_ref 'heads/foo\bar' # should fail due to \ +invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB +invalid_ref "$(printf 'heads/foo\177')" +invalid_ref '@' + +invalid_ref 'foo/*' +invalid_ref '*/foo' +invalid_ref 'foo/*/bar' +invalid_ref '*' +invalid_ref 'foo/*/*' +invalid_ref '*/foo/*' +invalid_ref '/foo' +invalid_ref '' diff --git a/tests/local.mk b/tests/local.mk index 56e5640ca..536661af8 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -18,6 +18,7 @@ nix_tests = \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ + fetchGitRefs.sh \ fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ From fb38459d6e58508245553380cccc03c0dbaa1542 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Sat, 30 May 2020 12:33:38 +0200 Subject: [PATCH 294/350] Ensure we restrict refspec interpretation while fetching As `git fetch` may chose to interpret refspec to it's liking, ensure that we only pass refs that begin with `refs/` as is, otherwise, prepend them with `refs/heads`. Otherwise, branches named `heads/foo` (I know it's bad, but it's allowed), would be fetched as `foo`, instead of `heads/foo`. --- src/libfetchers/git.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e069057eb..75ce5ee8b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -282,7 +282,10 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 + ? *input->ref + : "refs/heads/" + *input->ref; + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); From eca1ff7a9f784926c52533c81cb5403a2cb804d3 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Sun, 31 May 2020 01:05:05 +0200 Subject: [PATCH 295/350] Add tests for lru-cache.hh --- src/libutil/tests/lru-cache.cc | 130 +++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/libutil/tests/lru-cache.cc diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc new file mode 100644 index 000000000..598977a6d --- /dev/null +++ b/src/libutil/tests/lru-cache.cc @@ -0,0 +1,130 @@ +#include "lru-cache.hh" +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * size + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, sizeOfEmptyCacheIsZero) { + LRUCache c(10); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, sizeOfSingleElementCacheIsOne) { + LRUCache c(10); + c.upsert("foo", "bar"); + ASSERT_EQ(c.size(), 1); + } + + /* ---------------------------------------------------------------------------- + * upsert / get + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, getFromEmptyCache) { + LRUCache c(10); + auto val = c.get("x"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, getExistingValue) { + LRUCache c(10); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val, "bar"); + } + + TEST(LRUCache, getNonExistingValueFromNonEmptyCache) { + LRUCache c(10); + c.upsert("foo", "bar"); + auto val = c.get("another"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, upsertOnZeroCapacityCache) { + LRUCache c(0); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, updateExistingValue) { + LRUCache c(1); + c.upsert("foo", "bar"); + + auto val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "bar"); + ASSERT_EQ(c.size(), 1); + + c.upsert("foo", "changed"); + val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "changed"); + ASSERT_EQ(c.size(), 1); + } + + TEST(LRUCache, overwriteOldestWhenCapacityIsReached) { + LRUCache c(3); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + + ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + + // exceed capacity + c.upsert("another", "whatever"); + + ASSERT_EQ(c.size(), 3); + // Retrieving "one" makes it the most recent element thus + // two will be the oldest one and thus replaced. + ASSERT_EQ(c.get("two").has_value(), false); + ASSERT_EQ(c.get("another").value(), "whatever"); + } + + /* ---------------------------------------------------------------------------- + * clear + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, clearEmptyCache) { + LRUCache c(10); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, clearNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + ASSERT_EQ(c.size(), 3); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + /* ---------------------------------------------------------------------------- + * erase + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, eraseFromEmptyCache) { + LRUCache c(10); + c.erase("foo"); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, eraseMissingFromNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + c.erase("foo"); + ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + } + + TEST(LRUCache, eraseFromNonEmptyCache) { + LRUCache c(10); + c.upsert("one", "eins"); + c.erase("one"); + ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.get("one").value_or("empty"), "empty"); + } +} From e9fee8e6a7ff8540a04620e7d75a987cfe89ab00 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Jun 2020 12:06:59 +0200 Subject: [PATCH 296/350] src/libutil/tests/lru-cache.cc: Check erase() Co-authored-by: James Lee --- src/libutil/tests/lru-cache.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc index 598977a6d..091d3d5ed 100644 --- a/src/libutil/tests/lru-cache.cc +++ b/src/libutil/tests/lru-cache.cc @@ -108,14 +108,14 @@ namespace nix { TEST(LRUCache, eraseFromEmptyCache) { LRUCache c(10); - c.erase("foo"); + ASSERT_EQ(c.erase("foo"), false); ASSERT_EQ(c.size(), 0); } TEST(LRUCache, eraseMissingFromNonEmptyCache) { LRUCache c(10); c.upsert("one", "eins"); - c.erase("foo"); + ASSERT_EQ(c.erase("foo"), false); ASSERT_EQ(c.size(), 1); ASSERT_EQ(c.get("one").value_or("error"), "eins"); } @@ -123,7 +123,7 @@ namespace nix { TEST(LRUCache, eraseFromNonEmptyCache) { LRUCache c(10); c.upsert("one", "eins"); - c.erase("one"); + ASSERT_EQ(c.erase("one"), true); ASSERT_EQ(c.size(), 0); ASSERT_EQ(c.get("one").value_or("empty"), "empty"); } From d82d230b4015c50e698e3be1ec7a3e12277a3251 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 2 Jun 2020 08:22:24 -0600 Subject: [PATCH 297/350] elide the 'ErrorInfo' in logError and logWarning calls --- src/build-remote/build-remote.cc | 11 +- src/error-demo/error-demo.cc | 111 +++++++++--------- src/libstore/build.cc | 107 ++++++++--------- src/libstore/filetransfer.cc | 11 +- src/libstore/local-store.cc | 45 ++++--- src/libstore/optimise-store.cc | 21 ++-- src/libutil/logging.cc | 7 +- src/nix-env/nix-env.cc | 27 ++--- src/nix-store/nix-store.cc | 17 ++- src/nix/verify.cc | 26 ++-- .../resolve-system-dependencies.cc | 28 ++--- 11 files changed, 187 insertions(+), 224 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 0caa97fa4..2f0b8c825 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -200,12 +200,11 @@ static int _main(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - logError( - ErrorInfo { - .name = "Remote build", - .hint = hintfmt("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)) + logError({ + .name = "Remote build", + .hint = hintfmt("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + (msg.empty() ? "" : ": " + msg)) }); bestMachine->enabled = false; continue; diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc index 0216092e3..f1e17b3de 100644 --- a/src/error-demo/error-demo.cc +++ b/src/error-demo/error-demo.cc @@ -62,101 +62,96 @@ int main() // For completeness sake, show 'info' through 'vomit' levels. // But this is maybe a heavy format for those. - logger->logEI( - ErrorInfo { .level = lvlInfo, + logger->logEI({ .level = lvlInfo, .name = "Info name", .description = "Info description", }); - logger->logEI( - ErrorInfo { .level = lvlTalkative, + logger->logEI({ .level = lvlTalkative, .name = "Talkative name", .description = "Talkative description", }); - logger->logEI( - ErrorInfo { .level = lvlChatty, + logger->logEI({ .level = lvlChatty, .name = "Chatty name", .description = "Chatty description", }); - logger->logEI( - ErrorInfo { .level = lvlDebug, + logger->logEI({ .level = lvlDebug, .name = "Debug name", .description = "Debug description", }); - logger->logEI( - ErrorInfo { .level = lvlVomit, + logger->logEI({ .level = lvlVomit, .name = "Vomit name", .description = "Vomit description", }); // Error in a program; no hint and no nix code. - logError( - ErrorInfo { .name = "name", - .description = "error description", - }); + logError({ + .name = "name", + .description = "error description", + }); // Warning with name, description, and hint. // The hintfmt function makes all the substituted text yellow. - logWarning( - ErrorInfo { .name = "name", - .description = "error description", - .hint = hintfmt("there was a %1%", "warning"), - }); + logWarning({ + .name = "name", + .description = "error description", + .hint = hintfmt("there was a %1%", "warning"), + }); // Warning with nix file, line number, column, and the lines of // code where a warning occurred. SymbolTable testTable; auto problem_file = testTable.create("myfile.nix"); - logWarning( - ErrorInfo { .name = "warning name", - .description = "warning description", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::nullopt, - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::nullopt - }}); + logWarning({ + .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); // Error with previous and next lines of code. - logError( - ErrorInfo { .name = "error name", - .description = "error with code lines", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = "previous line of code", - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = "next line of code", - }}); + logError({ + .name = "error name", + .description = "error with code lines", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = "previous line of code", + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = "next line of code", + }}); // Error without lines of code. - logError( - ErrorInfo { .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) - }}); + logError({ + .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); // Error with only hint and name.. - logError( - ErrorInfo { .name = "error name", - .hint = hintfmt("hint %1%", "only"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) - }}); + logError({ + .name = "error name", + .hint = hintfmt("hint %1%", "only"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); return 0; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4630432c9..04e8d2ebe 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1158,10 +1158,9 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - logError( - ErrorInfo { - .name = "missing derivation during build", - .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) + logError({ + .name = "missing derivation during build", + .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) }); done(BuildResult::MiscFailure); return; @@ -1313,12 +1312,11 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - logError( - ErrorInfo { - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + logError({ + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) }); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) @@ -1353,12 +1351,11 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - logError( - ErrorInfo { - .name = "Dependencies could not be built", - .hint = hintfmt( - "cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed) + logError({ + .name = "Dependencies could not be built", + .hint = hintfmt( + "cannot build derivation '%s': %s dependencies couldn't be built", + worker.store.printStorePath(drvPath), nrFailed) }); done(BuildResult::DependencyFailed); return; @@ -1844,12 +1841,11 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - logError( - ErrorInfo { - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) + logError({ + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) }); worker.hook = 0; return rpDecline; @@ -3692,11 +3688,10 @@ void DerivationGoal::registerOutputs() /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - logWarning( - ErrorInfo { - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) - }); + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -3875,10 +3870,9 @@ void DerivationGoal::registerOutputs() if (settings.enforceDeterminism) throw NotDeterministic(hint); - logError( - ErrorInfo { - .name = "Output determinism error", - .hint = hint + logError({ + .name = "Output determinism error", + .hint = hint }); @@ -4145,12 +4139,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - logError( - ErrorInfo { - .name = "Max log size exceeded", - .hint = hintfmt( - "%1% killed after writing more than %2% bytes of log output", - getName(), settings.maxLogSize) + logError({ + .name = "Max log size exceeded", + .hint = hintfmt( + "%1% killed after writing more than %2% bytes of log output", + getName(), settings.maxLogSize) }); killChild(); done(BuildResult::LogLimitExceeded); @@ -4467,12 +4460,11 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - logWarning( - ErrorInfo { - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); + logWarning({ + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } @@ -4954,12 +4946,11 @@ void Worker::waitForInput() j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - logError( - ErrorInfo { - .name = "Silent build timeout", - .hint = hintfmt( - "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime) + logError({ + .name = "Silent build timeout", + .hint = hintfmt( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime) }); goal->timedOut(); } @@ -4969,12 +4960,11 @@ void Worker::waitForInput() j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - logError( - ErrorInfo { - .name = "Build timeout", - .hint = hintfmt( - "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout) + logError({ + .name = "Build timeout", + .hint = hintfmt( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout) }); goal->timedOut(); } @@ -5035,10 +5025,9 @@ bool Worker::pathContentsGood(const StorePath & path) } pathContentsGoodCache.insert_or_assign(path.clone(), res); if (!res) - logError( - ErrorInfo { - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + logError({ + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) }); return res; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 89588d662..6ca3393ab 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -599,12 +599,11 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - logError( - ErrorInfo { - .name = "File transfer", - .hint = hintfmt("unexpected error in download thread: %s", - e.what()) - }); + logError({ + .name = "File transfer", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); } { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index abcfe5271..53fb4ce68 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,12 +87,11 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - logError( - ErrorInfo { - .name = "'build-users-group' not found", - .hint = hintfmt( - "warning: the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup) + logError({ + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) }); else { struct stat st; @@ -1242,12 +1241,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - logError( - ErrorInfo { - .name = "Invalid hash", - .hint = hintfmt( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash) + logError({ + .name = "Invalid hash", + .hint = hintfmt( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash) }); if (repair) { if (unlink(linkPath.c_str()) == 0) @@ -1281,11 +1279,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - logError( - ErrorInfo { - .name = "Invalid hash - path modified", - .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()) + logError({ + .name = "Invalid hash - path modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(), current.first.to_string()) }); if (repair) repairPath(i); else errors = true; } else { @@ -1337,10 +1334,9 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - logError( - ErrorInfo { - .name = "Nix path not found", - .hint = hintfmt("path '%s' is not in the Nix store", pathS) + logError({ + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) }); return; } @@ -1364,10 +1360,9 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - logError( - ErrorInfo { - .name = "Missing path with referrers", - .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + logError({ + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) }); if (repair) try { diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c9ecca5a8..cc0507be2 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -130,10 +130,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - logWarning( - ErrorInfo { - .name = "Suspicious file", - .hint = hintfmt("skipping suspicious writable file '%1%'", path) + logWarning({ + .name = "Suspicious file", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) }); return; } @@ -198,10 +197,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - logWarning( - ErrorInfo { - .name = "Corrupted link", - .hint = hintfmt("removing corrupted link '%1%'", linkPath) + logWarning({ + .name = "Corrupted link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) }); unlink(linkPath.c_str()); goto retry; @@ -237,10 +235,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - logError( - ErrorInfo { - .name = "Unlink error", - .hint = hintfmt("unable to unlink '%1%'", tempLink) + logError({ + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 071a7ec7e..41378b0db 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -251,10 +251,9 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - logError( - ErrorInfo { - .name = "Json log message", - .hint = hintfmt("bad log message from builder: %s", e.what()) + logError({ + .name = "Json log message", + .hint = hintfmt("bad log message from builder: %s", e.what()) }); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index bbce5dcda..8231a07a4 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -123,10 +123,9 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - logError( - ErrorInfo { - .name = "Name collision", - .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) + logError({ + .name = "Name collision", + .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) }); continue; } @@ -875,11 +874,10 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - logError( - ErrorInfo { - .name = "Invalid meta attribute", - .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) }); placeholder.write(nullptr); } else { @@ -1131,12 +1129,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["name"] = j; Value * v = i.queryMeta(j); if (!v) - logError( - ErrorInfo { - .name = "Invalid meta attribute", - .hint = hintfmt( - "derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j) + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) }); else { if (v->type == tString) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6b83093b1..5e42736fc 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -704,7 +704,7 @@ static void opVerify(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - logWarning(ErrorInfo { + logWarning({ .name = "Store consistency", .description = "not all errors were fixed" }); @@ -729,14 +729,13 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - logError( - ErrorInfo { - .name = "Hash mismatch", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), - info->narHash.to_string(), - current.first.to_string()) + logError({ + .name = "Hash mismatch", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(), + current.first.to_string()) }); status = 1; } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 8c845bfc2..8ecd9a8f3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -99,15 +99,14 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - logError( - ErrorInfo { - .name = "Hash error - path modified", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), - info->narHash.to_string(), - hash.first.to_string()) - }); + logError({ + .name = "Hash error - path modified", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(), + hash.first.to_string()) + }); } } @@ -156,11 +155,10 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - logError( - ErrorInfo { - .name = "Untrusted path", - .hint = hintfmt("path '%s' is untrusted", - store->printStorePath(info->path)) + logError({ + .name = "Untrusted path", + .hint = hintfmt("path '%s' is untrusted", + store->printStorePath(info->path)) }); } diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index dcea72529..82feacb3d 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,19 +39,17 @@ std::set runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - logError( - ErrorInfo { - .name = "Regular MACH file", - .hint = hintfmt("file '%s' is not a regular file", filename) + logError({ + .name = "Regular MACH file", + .hint = hintfmt("file '%s' is not a regular file", filename) }); return {}; } if (st.st_size < sizeof(mach_header_64)) { - logError( - ErrorInfo { - .name = "File too short", - .hint = hintfmt("file '%s' is too short for a MACH binary", filename) + logError({ + .name = "File too short", + .hint = hintfmt("file '%s' is too short for a MACH binary", filename) }); return {}; } @@ -74,20 +72,18 @@ std::set runResolver(const Path & filename) } } if (mach64_offset == 0) { - logError( - ErrorInfo { - .name = "No mach64 blobs", - .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) + logError({ + .name = "No mach64 blobs", + .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) }); return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - logError( - ErrorInfo { - .name = "Magic number", - .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) + logError({ + .name = "Magic number", + .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) }); return {}; } From 156d4f8bc892baa4e25814d5f79a332994c750d1 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 2 Jun 2020 08:45:37 -0600 Subject: [PATCH 298/350] remove extra space in SysErrors --- src/libutil/error.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index b374c2780..0ca0c8b15 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -155,7 +155,7 @@ public: { errNo = errno; auto hf = hintfmt(args...); - err.hint = hintfmt("%1% : %2%", normaltxt(hf.str()), strerror(errNo)); + err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } virtual const char* sname() const override { return "SysError"; } From 01572c2198de49071827f0be9f5db202bac21703 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Jun 2020 06:15:22 -0400 Subject: [PATCH 299/350] Missing `#include ` in `lru-cache.hh` (#3654) This was a latent bug that just appeared because of the tests that were added. Remember to wait for CI! :) --- src/libutil/lru-cache.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 8b83f842c..6ef4a3e06 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include From 6ee03b8444e1838b9985a9fd4f0f46947e958b3b Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 3 Jun 2020 12:38:23 +0200 Subject: [PATCH 300/350] libutils/hash: remove default encoding This will make it easier to reason about the hash encoding and switch to SRI everywhere where possible. --- perl/lib/Nix/Store.xs | 4 ++-- src/libexpr/primops/fetchTree.cc | 4 ++-- src/libfetchers/fetchers.cc | 4 ++-- src/libfetchers/tarball.cc | 6 +++--- src/libstore/build.cc | 2 +- src/libstore/export-import.cc | 2 +- src/libstore/local-store.cc | 10 +++++----- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- src/libstore/store-api.cc | 12 ++++++------ src/libutil/hash.hh | 2 +- src/libutil/tests/hash.cc | 16 ++++++++-------- src/nix-store/nix-store.cc | 6 +++--- src/nix/verify.cc | 2 +- 15 files changed, 40 insertions(+), 40 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 890310b3e..945ed49c7 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16); + auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c5a0d9886..b90408798 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -23,7 +23,7 @@ void emitTreeAttrs( assert(tree.info.narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + tree.info.narHash.to_string(SRI, true)); if (input->getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); @@ -140,7 +140,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(), hash.to_string()); + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 94ac30e38..ca555114d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -47,7 +47,7 @@ Attrs Input::toAttrs() const { auto attrs = toAttrsInternal(); if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("narHash", narHash->to_string(SRI, true)); attrs.emplace("type", type()); return attrs; } @@ -67,7 +67,7 @@ std::pair> Input::fetchTree(ref store) if (narHash && narHash != input->narHash) throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); return {std::move(tree), input}; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6e57379b..1b0eba18a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -196,9 +196,9 @@ struct TarballInput : Input // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); + url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); return url2; } @@ -207,7 +207,7 @@ struct TarballInput : Input Attrs attrs; attrs.emplace("url", url.to_string()); if (hash) - attrs.emplace("hash", hash->to_string(SRI)); + attrs.emplace("hash", hash->to_string(SRI, true)); return attrs; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..9582a9007 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3718,7 +3718,7 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index f0d01a240..54471d4a3 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -57,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..1c3795eb1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -580,7 +580,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -680,7 +680,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -1022,7 +1022,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1155,7 +1155,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1273,7 +1273,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 442541330..e8cf1d177 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..6b16be08a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..3f4b72b9c 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -150,7 +150,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..0645fca84 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -142,7 +142,7 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); return StorePath::make(h.hash, name); } @@ -186,7 +186,7 @@ StorePath Store::makeFixedOutputPath( hashString(htSHA256, "fixed:out:" + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") - + hash.to_string(Base16) + ":"), + + hash.to_string(Base16, true) + ":"), name); } } @@ -461,7 +461,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -504,7 +504,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -760,7 +760,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -836,7 +836,7 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { return "fixed:" + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + + hash.to_string(Base32, true); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ea9fca3e7..4ff13f6b2 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -79,7 +79,7 @@ struct Hash /* Return a string representation of the hash, in base-16, base-32 or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ - std::string to_string(Base base = Base32, bool includeType = true) const; + std::string to_string(Base base, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 7cb439817..5334b046e 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -11,28 +11,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -40,7 +40,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -48,7 +48,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -56,7 +56,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -67,7 +67,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad8..9491a0c26 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -373,7 +373,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32)); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -728,7 +728,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); + store->printStorePath(path), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); status = 1; } } @@ -856,7 +856,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index cf1fa6a99..287dad101 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -101,7 +101,7 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); + store->printStorePath(info->path), info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true)); } } From f97576c5d91d5278155a67e2b8982d2233d4335a Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 3 Jun 2020 14:47:00 -0600 Subject: [PATCH 301/350] newline-as-prefix; no final newline in output. --- src/libutil/error.cc | 73 +++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 59628d875..539a7543f 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -57,20 +57,20 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC { // previous line of code. if (nixCode.prevLineOfCode.has_value()) { - out << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line - 1), - *nixCode.prevLineOfCode) - << std::endl; + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line - 1), + *nixCode.prevLineOfCode); } if (nixCode.errLineOfCode.has_value()) { // line of code containing the error. - out << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line), - *nixCode.errLineOfCode) - << std::endl; + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line), + *nixCode.errLineOfCode); // error arrows for the column range. if (nixCode.errPos.column > 0) { int start = nixCode.errPos.column; @@ -81,20 +81,21 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC std::string arrows("^"); - out << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, - prefix, - spaces, - arrows) << std::endl; + out << std::endl + << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows); } } // next line of code. if (nixCode.nextLineOfCode.has_value()) { - out << fmt("%1% %|2$5d|| %3%", - prefix, - (nixCode.errPos.line + 1), - *nixCode.nextLineOfCode) - << std::endl; + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line + 1), + *nixCode.nextLineOfCode); } } @@ -167,46 +168,50 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) levelString, einfo.name, dashes, - einfo.programName.value_or("")) - << std::endl; + einfo.programName.value_or("")); else out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, prefix, levelString, dashes, - einfo.programName.value_or("")) - << std::endl; + einfo.programName.value_or("")); - // filename, line, column. + bool nl = false; // intersperse newline between sections. if (einfo.nixCode.has_value()) { if (einfo.nixCode->errPos.file != "") { - out << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, + // filename, line, column. + out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, prefix, einfo.nixCode->errPos.file, - showErrPos(einfo.nixCode->errPos)) << std::endl; - out << prefix << std::endl; + showErrPos(einfo.nixCode->errPos)); } else { - out << fmt("%1%from command line argument", prefix) << std::endl; - out << prefix << std::endl; + out << std::endl << fmt("%1%from command line argument", prefix); } + nl = true; } // description if (einfo.description != "") { - out << prefix << einfo.description << std::endl; - out << prefix << std::endl; + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << einfo.description; + nl = true; } // lines of code. if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) { + if (nl) + out << std::endl << prefix; printCodeLines(out, prefix, *einfo.nixCode); - out << prefix << std::endl; + nl = true; } // hint if (einfo.hint.has_value()) { - out << prefix << *einfo.hint << std::endl; - out << prefix << std::endl; + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << *einfo.hint; + nl = true; } return out; From 721943e1d44b63d1a4055309fad39e1f4c0f9f0d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 3 Jun 2020 17:32:57 -0600 Subject: [PATCH 302/350] update error grep --- tests/fetchGitRefs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh index 93993ae90..23934698e 100644 --- a/tests/fetchGitRefs.sh +++ b/tests/fetchGitRefs.sh @@ -56,7 +56,7 @@ invalid_ref() { else (! git check-ref-format --branch "$1" >/dev/null 2>&1) fi - nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'error: invalid Git branch/tag name' >/dev/null + nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null } From 61e3d598b67290ca66f34fcac7b7373f506240da Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jun 2020 10:57:40 +0200 Subject: [PATCH 303/350] Rename 'nix dev-shell' to 'nix develop' Fixes #3648. --- src/nix/{dev-shell.cc => develop.cc} | 14 +++++++------- src/nix/local.mk | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename src/nix/{dev-shell.cc => develop.cc} (96%) diff --git a/src/nix/dev-shell.cc b/src/nix/develop.cc similarity index 96% rename from src/nix/dev-shell.cc rename to src/nix/develop.cc index d300f6a23..3045d7dc3 100644 --- a/src/nix/dev-shell.cc +++ b/src/nix/develop.cc @@ -110,7 +110,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) auto builder = baseNameOf(drv.builder); if (builder != "bash") - throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder"); + throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); @@ -231,11 +231,11 @@ struct Common : InstallableCommand, MixProfile } }; -struct CmdDevShell : Common, MixEnvironment +struct CmdDevelop : Common, MixEnvironment { std::vector command; - CmdDevShell() + CmdDevelop() { addFlag({ .longName = "command", @@ -259,15 +259,15 @@ struct CmdDevShell : Common, MixEnvironment return { Example{ "To get the build environment of GNU hello:", - "nix dev-shell nixpkgs.hello" + "nix develop nixpkgs.hello" }, Example{ "To store the build environment in a profile:", - "nix dev-shell --profile /tmp/my-shell nixpkgs.hello" + "nix develop --profile /tmp/my-shell nixpkgs.hello" }, Example{ "To use a build environment previously recorded in a profile:", - "nix dev-shell /tmp/my-shell" + "nix develop /tmp/my-shell" }, }; } @@ -341,4 +341,4 @@ struct CmdPrintDevEnv : Common }; static auto r1 = registerCommand("print-dev-env"); -static auto r2 = registerCommand("dev-shell"); +static auto r2 = registerCommand("develop"); diff --git a/src/nix/local.mk b/src/nix/local.mk index 8c0eed19e..43b7754e3 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -28,4 +28,4 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh +src/nix/develop.cc: src/nix/get-env.sh.gen.hh From 0f44b60e6dc999697bf9f2a4b3652a0551016598 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jun 2020 11:14:19 +0200 Subject: [PATCH 304/350] Make 'nix dev-shell' a deprecated alias for 'nix develop' --- src/libutil/args.cc | 11 ++++++++--- src/libutil/args.hh | 2 ++ src/libutil/util.hh | 3 +-- src/nix/main.cc | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index f829415d1..afeaf4cea 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -217,10 +217,15 @@ MultiCommand::MultiCommand(const Commands & commands) { expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { assert(!command); - auto i = commands.find(ss[0]); + auto cmd = ss[0]; + if (auto alias = get(deprecatedAliases, cmd)) { + warn("'%s' is a deprecated alias for '%s'", cmd, *alias); + cmd = *alias; + } + auto i = commands.find(cmd); if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", ss[0]); - command = {ss[0], i->second()}; + throw UsageError("'%s' is not a recognised command", cmd); + command = {cmd, i->second()}; }}); categories[Command::catDefault] = "Available commands"; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1932e6a8a..154d1e6aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -234,6 +234,8 @@ public: std::map categories; + std::map deprecatedAliases; + // Selected command, if any. std::optional>> command; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a63ee05b3..4b117f9bc 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -459,8 +459,7 @@ string base64Encode(const string & s); string base64Decode(const string & s); -/* Get a value for the specified key from an associate container, or a - default value if the key doesn't exist. */ +/* Get a value for the specified key from an associate container. */ template std::optional get(const T & map, const typename T::key_type & key) { diff --git a/src/nix/main.cc b/src/nix/main.cc index ef301580a..1120ba5ef 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -110,6 +110,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "consider all previously downloaded files out-of-date", .handler = {[&]() { refresh = true; }}, }); + + deprecatedAliases.insert({"dev-shell", "develop"}); } void printFlags(std::ostream & out) override From 2299ef705c2d69add371ee4ccca04de9f2628853 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 3 Jun 2020 15:18:46 -0500 Subject: [PATCH 305/350] Add error message when FileIngestionMethod is out of bounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bool coerces anything >0 to true, but in the future we may have other file ingestion methods. This shows a better error message when the “recursive” byte isn’t 1. --- src/libstore/daemon.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f1afdff69..5cff170dd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -358,8 +358,10 @@ static void performOp(TunnelLogger * logger, ref store, std::string s, baseName; FileIngestionMethod method; { - bool fixed, recursive; + bool fixed; uint8_t recursive; from >> baseName >> fixed /* obsolete */ >> recursive >> s; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { From 94427ffee3493077206f960e3cda9bbb659583be Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Thu, 4 Jun 2020 11:53:19 -0600 Subject: [PATCH 306/350] add some comments --- src/libutil/error.cc | 6 +++++- src/libutil/error.hh | 21 +++++++++++++++++++-- src/libutil/logging.hh | 33 ++++++++++++++++++--------------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 539a7543f..1fcb8111c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -10,13 +10,16 @@ namespace nix { const std::string nativeSystem = SYSTEM; - +// addPrefix is used for show-trace. Strings added with addPrefix +// will print ahead of the error itself. BaseError & BaseError::addPrefix(const FormatOrString & fs) { prefix_ = fs.s + prefix_; return *this; } +// c++ std::exception descendants must have a 'const char* what()' function. +// This stringifies the error and caches it for use by what(), or similarly by msg(). const string& BaseError::calcWhat() const { if (what_.has_value()) @@ -53,6 +56,7 @@ string showErrPos(const ErrPos &errPos) } } +// if nixCode contains lines of code, print them to the ostream, indicating the error column. void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode) { // previous line of code. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0ca0c8b15..8a48fa105 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -22,6 +22,23 @@ namespace nix { +/* + +This file defines two main structs/classes used in nix error handling. + +ErrorInfo provides a standard payload of error information, with conversion to string +happening in the logger rather than at the call site. + +BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains +an ErrorInfo. + +ErrorInfo structs are sent to the logger as part of an exception, or directly with the +logError or logWarning macros. + +See the error-demo.cc program for usage examples. + +*/ + typedef enum { lvlError = 0, lvlWarn, @@ -32,6 +49,7 @@ typedef enum { lvlVomit } Verbosity; +// ErrPos indicates the location of an error in a nix file. struct ErrPos { int line = 0; int column = 0; @@ -42,6 +60,7 @@ struct ErrPos { return line != 0; } + // convert from the Pos struct, found in libexpr. template ErrPos& operator=(const P &pos) { @@ -65,8 +84,6 @@ struct NixCode { std::optional nextLineOfCode; }; -// ------------------------------------------------- -// ErrorInfo. struct ErrorInfo { Verbosity level; string name; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 39692a291..eeb7233e9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -150,9 +150,23 @@ bool handleJSONLogMessage(const std::string & msg, extern Verbosity verbosity; /* suppress msgs > this */ -/* Print a message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure - that the arguments are evaluated lazily. */ +/* Print a message with the standard ErrorInfo format. + In general, use these 'log' macros for reporting problems that may require user + intervention or that need more explanation. Use the 'print' macros for more + lightweight status messages. */ +#define logErrorInfo(level, errorInfo...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->logEI(level, errorInfo); \ + } \ + } while (0) + +#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) +#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) + +/* Print a string message if the current log level is at least the specified + level. Note that this has to be implemented as a macro to ensure that the + arguments are evaluated lazily. */ #define printMsg(level, args...) \ do { \ if (level <= nix::verbosity) { \ @@ -166,18 +180,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -#define logErrorInfo(level, errorInfo...) \ - do { \ - if (level <= nix::verbosity) { \ - logger->logEI(level, errorInfo); \ - } \ - } while (0) - -#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) -#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) - - - +/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ template inline void warn(const std::string & fs, const Args & ... args) { From efc5e45e95734cdade04ad6231c5185a298357d0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 4 Jun 2020 21:05:41 +0000 Subject: [PATCH 307/350] Add `src/libutil/tests/libutil-tests` to `.gitignore` I gather this comes from the new unit tests. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ad5684123..983026570 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,9 @@ perl/Makefile.config # /src/libstore/ *.gen.* +# /src/libutil/ +/src/libutil/tests/libutil-tests + /src/nix/nix # /src/nix-env/ From 39e84c35d0b8bf5b49f45736febd9f506e3ee4ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jun 2020 10:45:05 +0200 Subject: [PATCH 308/350] Fix log-prefix of `nix build -L` Alternative fix to #3661. The cause was that 'name' is a std::string_view into a temporary which could get overwritten. --- src/nix/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 8e7ba95a3..c67701098 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -154,7 +154,7 @@ public: state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - auto name = storePathToName(getS(fields, 0)); + std::string name(storePathToName(getS(fields, 0))); if (hasSuffix(name, ".drv")) name = name.substr(0, name.size() - 4); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); From 952e72c804ce79ffeee8f58261d4cbdeb9d56ac6 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Sat, 6 Jun 2020 10:22:32 +0200 Subject: [PATCH 309/350] Add tests for logging.hh --- src/libutil/tests/local.mk | 2 +- src/libutil/tests/logging.cc | 251 +++++++++++++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 src/libutil/tests/logging.cc diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index a297edb64..815e18560 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -8,7 +8,7 @@ libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil +libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr libutil-tests_LIBS = libutil diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc new file mode 100644 index 000000000..ac015a16d --- /dev/null +++ b/src/libutil/tests/logging.cc @@ -0,0 +1,251 @@ +#include "logging.hh" +#include "nixexpr.hh" +#include "util.hh" + +#include + +namespace nix { + + /* ---------------------------------------------------------------------------- + * logEI + * --------------------------------------------------------------------------*/ + + TEST(logEI, catpuresBasicProperties) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + throw TestError("an error for testing purposes"); + } catch (Error &e) { + testing::internal::CaptureStderr(); + logger->logEI(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\nan error for testing purposes\n"); + } + } + + TEST(logEI, appendingHintsToPreviousError) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto e = Error("initial error"); + throw TestError(e.info()); + } catch (Error &e) { + ErrorInfo ei = e.info(); + ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : "")); + + testing::internal::CaptureStderr(); + logger->logEI(ei); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); + } + + } + + TEST(logEI, picksUpSysErrorExitCode) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto x = readFile(-1); + } + catch (SysError &e) { + testing::internal::CaptureStderr(); + logError(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError ------------------------------------- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); + + } + } + + TEST(logEI, loggingErrorOnInfoLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlInfo, + .name = "Info name", + .description = "Info description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name ------------------------------------- error-unit-test\x1B[0m\nInfo description\n"); + } + + TEST(logEI, loggingErrorOnTalkativeLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlTalkative, + .name = "Talkative name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + // XXX: why is this the empty string? + ASSERT_STREQ(str.c_str(), ""); + } + + TEST(logEI, loggingErrorOnChattyLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlChatty, + .name = "Chatty name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + // XXX: why is this the empty string? + ASSERT_STREQ(str.c_str(), ""); + } + + TEST(logEI, loggingErrorOnDebugLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlDebug, + .name = "Debug name", + .description = "Debug description", + }); + + auto str = testing::internal::GetCapturedStderr(); + // XXX: why is this the empty string? + ASSERT_STREQ(str.c_str(), ""); + } + + TEST(logEI, loggingErrorOnVomitLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlVomit, + .name = "Vomit name", + .description = "Vomit description", + }); + + auto str = testing::internal::GetCapturedStderr(); + // XXX: why is this the empty string? + ASSERT_STREQ(str.c_str(), ""); + } + + /* ---------------------------------------------------------------------------- + * logError + * --------------------------------------------------------------------------*/ + + + TEST(logError, logErrorWithoutHintOrCode) { + testing::internal::CaptureStderr(); + + logError({ + .name = "name", + .description = "error description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name ----------------------------------------- error-unit-test\x1B[0m\nerror description\n"); + } + + TEST(logError, logErrorWithPreviousAndNextLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error with code lines", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = "previous line of code", + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = "next line of code", + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithoutLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithOnlyHintAndName) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .hint = hintfmt("hint %1%", "only"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n"); + + } + + /* ---------------------------------------------------------------------------- + * logWarning + * --------------------------------------------------------------------------*/ + + TEST(logWarning, logWarningWithNameDescriptionAndHint) { + testing::internal::CaptureStderr(); + + logWarning({ + .name = "name", + .description = "error description", + .hint = hintfmt("there was a %1%", "warning"), + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --------------------------------------- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); + } + + TEST(logWarning, logWarningWithFileLineNumAndCode) { + + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logWarning({ + .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name ------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + +} From e60747b5fbe8b9c646ffe07ef8d28d949302efa6 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Sat, 6 Jun 2020 10:23:12 +0200 Subject: [PATCH 310/350] Remove error-demo/error-demo.cc The logging.hh superseeds the demo --- src/error-demo/error-demo.cc | 157 ----------------------------------- src/error-demo/local.mk | 12 --- 2 files changed, 169 deletions(-) delete mode 100644 src/error-demo/error-demo.cc delete mode 100644 src/error-demo/local.mk diff --git a/src/error-demo/error-demo.cc b/src/error-demo/error-demo.cc deleted file mode 100644 index f1e17b3de..000000000 --- a/src/error-demo/error-demo.cc +++ /dev/null @@ -1,157 +0,0 @@ -#include "logging.hh" -#include "nixexpr.hh" -#include "util.hh" - -#include -#include - -using namespace nix; - -MakeError(DemoError, Error); - -int main() -{ - verbosity = lvlVomit; - - // In each program where errors occur, this has to be set. - ErrorInfo::programName = std::optional("error-demo"); - - // 'DemoError' appears as error name. - try { - throw DemoError("demo error was thrown"); - } catch (Error &e) { - logger->logEI(e.info()); - } - - // appending to the hint from the previous error - try { - auto e = Error("initial error"); - throw DemoError(e.info()); - } catch (Error &e) { - ErrorInfo ei = e.info(); - // using normaltxt to avoid the default yellow highlighting. - ei.hint = hintfmt("%s; subsequent error message.", - normaltxt(e.info().hint ? e.info().hint->str() : "")); - logger->logEI(ei); - } - - // SysError; picks up errno - try { - auto x = readFile(-1); - } - catch (SysError &e) { - std::cout << "errno was: " << e.errNo << std::endl; - logError(e.info()); - } - - // current exception - try { - throw DemoError("DemoError handled as a %1%", "std::exception"); - } - catch (...) { - const std::exception_ptr &eptr = std::current_exception(); - try - { - std::rethrow_exception(eptr); - } - catch (std::exception& e) - { - std::cerr << e.what() << std::endl; - } - } - - // For completeness sake, show 'info' through 'vomit' levels. - // But this is maybe a heavy format for those. - logger->logEI({ .level = lvlInfo, - .name = "Info name", - .description = "Info description", - }); - - logger->logEI({ .level = lvlTalkative, - .name = "Talkative name", - .description = "Talkative description", - }); - - logger->logEI({ .level = lvlChatty, - .name = "Chatty name", - .description = "Chatty description", - }); - - logger->logEI({ .level = lvlDebug, - .name = "Debug name", - .description = "Debug description", - }); - - logger->logEI({ .level = lvlVomit, - .name = "Vomit name", - .description = "Vomit description", - }); - - // Error in a program; no hint and no nix code. - logError({ - .name = "name", - .description = "error description", - }); - - // Warning with name, description, and hint. - // The hintfmt function makes all the substituted text yellow. - logWarning({ - .name = "name", - .description = "error description", - .hint = hintfmt("there was a %1%", "warning"), - }); - - // Warning with nix file, line number, column, and the lines of - // code where a warning occurred. - SymbolTable testTable; - auto problem_file = testTable.create("myfile.nix"); - - logWarning({ - .name = "warning name", - .description = "warning description", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = std::nullopt, - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = std::nullopt - }}); - - // Error with previous and next lines of code. - logError({ - .name = "error name", - .description = "error with code lines", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13), - .prevLineOfCode = "previous line of code", - .errLineOfCode = "this is the problem line of code", - .nextLineOfCode = "next line of code", - }}); - - - // Error without lines of code. - logError({ - .name = "error name", - .description = "error without any code lines.", - .hint = hintfmt("this hint has %1% templated %2%!!", - "yellow", - "values"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) - }}); - - // Error with only hint and name.. - logError({ - .name = "error name", - .hint = hintfmt("hint %1%", "only"), - .nixCode = NixCode { - .errPos = Pos(problem_file, 40, 13) - }}); - - return 0; -} diff --git a/src/error-demo/local.mk b/src/error-demo/local.mk deleted file mode 100644 index 2c528490a..000000000 --- a/src/error-demo/local.mk +++ /dev/null @@ -1,12 +0,0 @@ -programs += error-demo - -error-demo_DIR := $(d) - -error-demo_SOURCES := \ - $(wildcard $(d)/*.cc) \ - -error-demo_CXXFLAGS += -I src/libutil -I src/libexpr - -error-demo_LIBS = libutil libexpr - -error-demo_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system From 94c347577ecea5dcd10a31ebfadf94db6ca5ab0d Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Sun, 7 Jun 2020 07:24:49 -0600 Subject: [PATCH 311/350] set verbosity levels --- src/libutil/tests/logging.cc | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index ac015a16d..fbdc91253 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -78,6 +78,8 @@ namespace nix { } TEST(logEI, loggingErrorOnTalkativeLevel) { + verbosity = lvlTalkative; + testing::internal::CaptureStderr(); logger->logEI({ .level = lvlTalkative, @@ -86,11 +88,12 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - // XXX: why is this the empty string? - ASSERT_STREQ(str.c_str(), ""); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name -------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); } TEST(logEI, loggingErrorOnChattyLevel) { + verbosity = lvlChatty; + testing::internal::CaptureStderr(); logger->logEI({ .level = lvlChatty, @@ -99,11 +102,12 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - // XXX: why is this the empty string? - ASSERT_STREQ(str.c_str(), ""); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name ----------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); } TEST(logEI, loggingErrorOnDebugLevel) { + verbosity = lvlDebug; + testing::internal::CaptureStderr(); logger->logEI({ .level = lvlDebug, @@ -112,11 +116,12 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - // XXX: why is this the empty string? - ASSERT_STREQ(str.c_str(), ""); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name ----------------------------------- error-unit-test\x1B[0m\nDebug description\n"); } TEST(logEI, loggingErrorOnVomitLevel) { + verbosity = lvlVomit; + testing::internal::CaptureStderr(); logger->logEI({ .level = lvlVomit, @@ -125,8 +130,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - // XXX: why is this the empty string? - ASSERT_STREQ(str.c_str(), ""); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name ----------------------------------- error-unit-test\x1B[0m\nVomit description\n"); } /* ---------------------------------------------------------------------------- From 170e86dff5724264e0d3d25b9af1bd42df6aec74 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Jun 2020 17:01:02 +0200 Subject: [PATCH 312/350] Make the logger customisable Add a new `--log-format` cli argument to change the format of the logs. The possible values are - raw (the default one for old-style commands) - bar (the default one for new-style commands) - bar-with-logs (equivalent to `--print-build-logs`) - internal-json (the internal machine-readable json format) --- src/libmain/common-args.cc | 9 ++++++++ src/libmain/loggers.cc | 47 ++++++++++++++++++++++++++++++++++++++ src/libmain/loggers.hh | 19 +++++++++++++++ src/libutil/logging.cc | 4 ++-- src/libutil/logging.hh | 4 +++- src/libutil/util.cc | 2 +- src/nix-build/nix-build.cc | 4 ++++ src/nix-env/nix-env.cc | 2 ++ src/nix-store/nix-store.cc | 2 ++ src/nix/main.cc | 11 +++++---- src/nix/progress-bar.cc | 14 ++++++++---- src/nix/progress-bar.hh | 2 ++ 12 files changed, 107 insertions(+), 13 deletions(-) create mode 100644 src/libmain/loggers.cc create mode 100644 src/libmain/loggers.hh diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 51e199ea5..a0cccceb4 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "globals.hh" +#include "loggers.hh" namespace nix { @@ -38,6 +39,14 @@ MixCommonArgs::MixCommonArgs(const string & programName) }}, }); + addFlag({ + .longName = "log-format", + .description = "Format of the logs. One of \"raw\", \"internal-json\", \"bar\" " + "or \"bar-with-logs\".", + .labels = {"format"}, + .handler = {[](std::string format) { setLogFormat(format); }}, + }); + addFlag({ .longName = "max-jobs", .shortName = 'j', diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc new file mode 100644 index 000000000..d3d5b104b --- /dev/null +++ b/src/libmain/loggers.cc @@ -0,0 +1,47 @@ +#include "loggers.hh" +#include "../nix/progress-bar.hh" + +namespace nix { + +LogFormat defaultLogFormat = LogFormat::raw; + +LogFormat parseLogFormat(const string &logFormatStr) { + if (logFormatStr == "raw") + return LogFormat::raw; + else if (logFormatStr == "internal-json") + return LogFormat::internalJson; + else if (logFormatStr == "bar") + return LogFormat::bar; + else if (logFormatStr == "bar-with-logs") + return LogFormat::barWithLogs; + throw Error(format("option 'log-format' has an invalid value '%s'") % + logFormatStr); +} + +Logger *makeDefaultLogger() { + switch (defaultLogFormat) { + case LogFormat::raw: + return makeSimpleLogger(); + case LogFormat::internalJson: + return makeJSONLogger(*makeSimpleLogger()); + case LogFormat::bar: + return makeProgressBar(); + case LogFormat::barWithLogs: + return makeProgressBar(true); + } +} + +void setLogFormat(const string &logFormatStr) { + setLogFormat(parseLogFormat(logFormatStr)); +} + +void setLogFormat(const LogFormat &logFormat) { + defaultLogFormat = logFormat; + createDefaultLogger(); +} + +void createDefaultLogger() { + logger = makeDefaultLogger(); +} + +} diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh new file mode 100644 index 000000000..f50cbb682 --- /dev/null +++ b/src/libmain/loggers.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +enum class LogFormat { + raw, + internalJson, + bar, + barWithLogs, +}; + +void setLogFormat(const string &logFormatStr); +void setLogFormat(const LogFormat &logFormat); + +void createDefaultLogger(); + +} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 3cc4ef8f1..6aec16e58 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -18,7 +18,7 @@ void setCurActivity(const ActivityId activityId) curActivity = activityId; } -Logger * logger = makeDefaultLogger(); +Logger * logger = makeSimpleLogger(); void Logger::warn(const std::string & msg) { @@ -94,7 +94,7 @@ void writeToStderr(const string & s) } } -Logger * makeDefaultLogger() +Logger * makeSimpleLogger() { return new SimpleLogger(); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 18c24d508..e319790fc 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -63,6 +63,8 @@ public: virtual ~Logger() { } + virtual void stop() { }; + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; void log(const FormatOrString & fs) @@ -141,7 +143,7 @@ struct PushActivity extern Logger * logger; -Logger * makeDefaultLogger(); +Logger * makeSimpleLogger(); Logger * makeJSONLogger(Logger & prevLogger); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 71db92d77..e0a99152b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -989,7 +989,7 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) { auto wrapper = [&]() { if (!options.allowVfork) - logger = makeDefaultLogger(); + logger = makeSimpleLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 0a058a31b..8649de5e9 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -472,6 +472,8 @@ static void _main(int argc, char * * argv) restoreSignals(); + logger->stop(); + execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -521,6 +523,8 @@ static void _main(int argc, char * * argv) if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); + logger->stop(); + for (auto & path : outPaths) std::cout << path << '\n'; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d62febaff..f7b04eb2b 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1446,6 +1446,8 @@ static int _main(int argc, char * * argv) globals.state->printStats(); + logger->stop(); + return 0; } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad8..708591b14 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -1098,6 +1098,8 @@ static int _main(int argc, char * * argv) op(opFlags, opArgs); + logger->stop(); + return 0; } } diff --git a/src/nix/main.cc b/src/nix/main.cc index 1120ba5ef..203901168 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -10,6 +10,7 @@ #include "progress-bar.hh" #include "filetransfer.hh" #include "finally.hh" +#include "loggers.hh" #include #include @@ -90,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "print-build-logs", .shortName = 'L', .description = "print full build logs on stderr", - .handler = {&printBuildLogs, true}, + .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ @@ -165,6 +166,10 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + setLogFormat("bar"); + + Finally f([] { logger->stop(); }); + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); @@ -178,10 +183,6 @@ void mainWrapped(int argc, char * * argv) && args.command->first != "upgrade-nix") settings.requireExperimentalFeature("nix-command"); - Finally f([]() { stopProgressBar(); }); - - startProgressBar(args.printBuildLogs); - if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); args.useNet = false; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index c67701098..828541bfe 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -106,7 +106,7 @@ public: updateThread.join(); } - void stop() + void stop() override { auto state(state_.lock()); if (!state->active) return; @@ -457,11 +457,17 @@ public: } }; +Logger *makeProgressBar(bool printBuildLogs) +{ + return new ProgressBar( + printBuildLogs, + isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + ); +} + void startProgressBar(bool printBuildLogs) { - logger = new ProgressBar( - printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"); + logger = makeProgressBar(printBuildLogs); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index 4d61175c2..60d0a2076 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -4,6 +4,8 @@ namespace nix { +Logger* makeProgressBar(bool printBuildLogs = false); + void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); From 2c4de6af1033367168320f43b0f04062bdac9234 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Thu, 26 Sep 2019 16:55:51 +0000 Subject: [PATCH 313/350] add documentation --- doc/manual/command-ref/opt-common-syn.xml | 6 ++++- doc/manual/command-ref/opt-common.xml | 31 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index b610b54b9..2660e3bb1 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -1,5 +1,5 @@ - + @@ -11,6 +11,10 @@ + + + format + diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 0383bfaed..a68eef1d0 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -92,6 +92,37 @@ + format + + + + This option can be used to change the output of the log format, with + format being one of: + + + + raw + This is the raw format, as outputted by nix-build. + + + internal-json + Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases. + + + bar + Only display a progress bar during the builds. + + + bar-with-logs + Display the raw logs, with the progress bar at the bottom. + + + + + + + + / By default, output written by builders to standard From 4983401440e1c46d6c576bc36ac86169bd296f9f Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Jun 2020 18:20:11 +0200 Subject: [PATCH 314/350] Unify the printing of the logs between bar-with-logs and raw Make the printing of the build logs systematically go through the logger, and replicate the behavior of `no-build-output` by having two different loggers (one that prints the build logs and one that doesn't) --- src/libmain/loggers.cc | 6 +++++- src/libmain/loggers.hh | 1 + src/libmain/shared.cc | 3 ++- src/libstore/build.cc | 17 ++++------------- src/libutil/logging.cc | 30 ++++++++++++++++++++++++++---- src/libutil/logging.hh | 5 ++++- src/nix/progress-bar.cc | 4 ++++ 7 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index d3d5b104b..b81096931 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -8,6 +8,8 @@ LogFormat defaultLogFormat = LogFormat::raw; LogFormat parseLogFormat(const string &logFormatStr) { if (logFormatStr == "raw") return LogFormat::raw; + else if (logFormatStr == "raw-with-logs") + return LogFormat::rawWithLogs; else if (logFormatStr == "internal-json") return LogFormat::internalJson; else if (logFormatStr == "bar") @@ -21,7 +23,9 @@ LogFormat parseLogFormat(const string &logFormatStr) { Logger *makeDefaultLogger() { switch (defaultLogFormat) { case LogFormat::raw: - return makeSimpleLogger(); + return makeSimpleLogger(false); + case LogFormat::rawWithLogs: + return makeSimpleLogger(true); case LogFormat::internalJson: return makeJSONLogger(*makeSimpleLogger()); case LogFormat::bar: diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f50cbb682..f9ba5ee5e 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -6,6 +6,7 @@ namespace nix { enum class LogFormat { raw, + rawWithLogs, internalJson, bar, barWithLogs, diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 70d1f0186..3bbb5cf93 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "util.hh" +#include "loggers.hh" #include #include @@ -169,7 +170,7 @@ LegacyArgs::LegacyArgs(const std::string & programName, .longName = "no-build-output", .shortName = 'Q', .description = "do not show build output", - .handler = {&settings.verboseBuild, false}, + .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..2d022093c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1642,7 +1642,7 @@ void DerivationGoal::buildDone() worker.store.printStorePath(drvPath), statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1691,11 +1691,7 @@ void DerivationGoal::buildDone() } void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(resPostBuildLogLine, currentLine); - } + act.result(resPostBuildLogLine, currentLine); currentLine.clear(); } @@ -4155,13 +4151,8 @@ void DerivationGoal::flushLine() ; else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); act->result(resBuildLogLine, currentLogLine); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 6aec16e58..15cbc1589 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -18,7 +18,7 @@ void setCurActivity(const ActivityId activityId) curActivity = activityId; } -Logger * logger = makeSimpleLogger(); +Logger * logger = makeSimpleLogger(true); void Logger::warn(const std::string & msg) { @@ -35,13 +35,19 @@ class SimpleLogger : public Logger public: bool systemd, tty; + bool printBuildLogs; - SimpleLogger() + SimpleLogger(bool printBuildLogs) + : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; tty = isatty(STDERR_FILENO); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { if (lvl > verbosity) return; @@ -70,6 +76,18 @@ public: if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (type == resBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError(lastLine); + } + else if (type == resPostBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError("post-build-hook: " + lastLine); + } + } }; Verbosity verbosity = lvlInfo; @@ -94,9 +112,9 @@ void writeToStderr(const string & s) } } -Logger * makeSimpleLogger() +Logger * makeSimpleLogger(bool printBuildLogs) { - return new SimpleLogger(); + return new SimpleLogger(printBuildLogs); } std::atomic nextId{(uint64_t) getpid() << 32}; @@ -114,6 +132,10 @@ struct JSONLogger : Logger JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + bool isVerbose() override { + return true; + } + void addFields(nlohmann::json & json, const Fields & fields) { if (fields.empty()) return; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index e319790fc..e3d91e01f 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -65,6 +65,9 @@ public: virtual void stop() { }; + // Whether the logger prints the whole build log + virtual bool isVerbose() { return false; } + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; void log(const FormatOrString & fs) @@ -143,7 +146,7 @@ struct PushActivity extern Logger * logger; -Logger * makeSimpleLogger(); +Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 828541bfe..c9ba89714 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -119,6 +119,10 @@ public: quitCV.notify_one(); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { auto state(state_.lock()); From f6ac888d3e1897f37a8accb9c46ebdf4fc2ca204 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Jun 2020 10:01:14 +0200 Subject: [PATCH 315/350] Actually test nix-env with a remote store The `remote-store` test loads the `user-env` one to test nix-env when using the daemon, but actually does it incorrectly because every test starts (in `common.sh`) by resetting the value of `NIX_REMOTE`, meaning that the `user-env` test will never use the daemon. Fix this by setting `NIX_REMOTE_` before sourcing `user-env.sh` in the `remote-store` test, so that `NIX_REMOTE` is correctly set inside the test --- tests/remote-store.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 77437658e..4cc73465a 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -4,7 +4,7 @@ clearStore startDaemon -storeCleared=1 $SHELL ./user-envs.sh +storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh nix-store --dump-db > $TEST_ROOT/d1 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 From c27f92698bbbb99df3c4403db6590276318940a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Jun 2020 13:24:01 +0200 Subject: [PATCH 316/350] Style fixes --- src/libmain/common-args.cc | 4 ++-- src/libmain/loggers.cc | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index a0cccceb4..051668e53 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -41,8 +41,8 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "Format of the logs. One of \"raw\", \"internal-json\", \"bar\" " - "or \"bar-with-logs\".", + .description = "format of log output; \"raw\", \"internal-json\", \"bar\" " + "or \"bar-with-logs\"", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index b81096931..ee3836382 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -16,8 +16,7 @@ LogFormat parseLogFormat(const string &logFormatStr) { return LogFormat::bar; else if (logFormatStr == "bar-with-logs") return LogFormat::barWithLogs; - throw Error(format("option 'log-format' has an invalid value '%s'") % - logFormatStr); + throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); } Logger *makeDefaultLogger() { From 801112de1af8e519e29e1e1c678f97de45a8c49d Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 8 Jun 2020 17:16:52 +0200 Subject: [PATCH 317/350] Move progress-bar.cc to libmain Needed so that we can include it as a logger in loggers.cc without adding a dependency on nix This also requires moving names.hh to libutil to prevent a circular dependency between libmain and libexpr --- src/libmain/loggers.cc | 2 +- src/{nix => libmain}/progress-bar.cc | 0 src/{nix => libmain}/progress-bar.hh | 0 src/{libexpr => libutil}/names.cc | 0 src/{libexpr => libutil}/names.hh | 0 src/nix-prefetch-url/nix-prefetch-url.cc | 2 +- 6 files changed, 2 insertions(+), 2 deletions(-) rename src/{nix => libmain}/progress-bar.cc (100%) rename src/{nix => libmain}/progress-bar.hh (100%) rename src/{libexpr => libutil}/names.cc (100%) rename src/{libexpr => libutil}/names.hh (100%) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index ee3836382..41f0b35fb 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,5 +1,5 @@ #include "loggers.hh" -#include "../nix/progress-bar.hh" +#include "progress-bar.hh" namespace nix { diff --git a/src/nix/progress-bar.cc b/src/libmain/progress-bar.cc similarity index 100% rename from src/nix/progress-bar.cc rename to src/libmain/progress-bar.cc diff --git a/src/nix/progress-bar.hh b/src/libmain/progress-bar.hh similarity index 100% rename from src/nix/progress-bar.hh rename to src/libmain/progress-bar.hh diff --git a/src/libexpr/names.cc b/src/libutil/names.cc similarity index 100% rename from src/libexpr/names.cc rename to src/libutil/names.cc diff --git a/src/libexpr/names.hh b/src/libutil/names.hh similarity index 100% rename from src/libexpr/names.hh rename to src/libutil/names.hh diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 5a686c8cd..b645bdc1b 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -8,7 +8,7 @@ #include "attr-path.hh" #include "finally.hh" #include "../nix/legacy.hh" -#include "../nix/progress-bar.hh" +#include "progress-bar.hh" #include "tarfile.hh" #include From 762273f1fd48b2b2f2bbedca65abfd4c07b5af05 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 9 Jun 2020 01:23:37 -0500 Subject: [PATCH 318/350] Allow empty hash in derivations follow up of https://github.com/NixOS/nix/pull/3544 This allows hash="" so that it can be used for debugging purposes. For instance, this gives you an error message like: warning: found empty hash, assuming you wanted 'sha256:0000000000000000000000000000000000000000000000000000' hash mismatch in fixed-output derivation '/nix/store/asx6qw1r1xk6iak6y6jph4n58h4hdmbm-nix': wanted: sha256:0000000000000000000000000000000000000000000000000000 got: sha256:0fpfhipl9v1mfzw2ffmxiyyzqwlkvww22bh9wcy4qrfslb4jm429 --- src/libexpr/primops.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d458ab272..de04fd2be 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -719,7 +719,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h(*outputHash, ht); + + Hash h; + if (outputHash->empty()) { + h = Hash(ht); + printError("warning: found empty hash, assuming you wanted '%s'", h.to_string()); + } else + h = Hash(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); From 19aa892f2064d437d32a3c12758d6a623b7ae8e5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 9 Jun 2020 11:10:54 -0500 Subject: [PATCH 319/350] Support empty hash in fetchers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchTarball, fetchTree, and fetchGit all have *optional* hash attrs. This means that we need to be careful with what we allow to avoid accidentally making these defaults. When ‘hash = ""’ we assume the empty hash is wanted. --- src/libexpr/primops.cc | 11 ++++++++--- src/libexpr/primops/fetchTree.cc | 11 ++++++++--- src/libfetchers/fetchers.cc | 11 ++++++++--- src/libfetchers/tarball.cc | 11 ++++++++--- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index de04fd2be..df3d4a459 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1131,9 +1131,14 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value filterFun = attr.value; } else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; - else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else + else if (n == "sha256") { + auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos); + if (hashStr == "") { + expectedHash = Hash(htSHA256); + printError("warning: found empty hash, assuming you wanted '%s'", expectedHash.to_string()); + } else + expectedHash = Hash(hashStr, htSHA256); + } else throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); } if (path.empty()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index c5a0d9886..745f65adf 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -102,9 +102,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, string n(attr.name); if (n == "url") url = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") + else if (n == "sha256") { + auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos); + if (hashStr == "") { + expectedHash = Hash(htSHA256); + printError("warning: found empty hash, assuming you wanted '%s'", expectedHash->to_string()); + } else + expectedHash = Hash(hashStr, htSHA256); + } else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 94ac30e38..988a8dc69 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -34,9 +34,14 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs2); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) { + if (narHash->empty()) { + res->narHash = Hash(htUnknown); + printError("warning: found empty hash, assuming you wanted '%s'", res->narHash->to_string()); + } else + // FIXME: require SRI hash. + res->narHash = Hash(*narHash); + } return res; } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6e57379b..e4dafec0b 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -263,9 +263,14 @@ struct TarballInputScheme : InputScheme throw Error("unsupported tarball input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); + if (auto hash = maybeGetStrAttr(attrs, "hash")) { + if (hash->empty()) { + input->hash = Hash(htUnknown); + printError("warning: found empty hash, assuming you wanted '%s'", input->hash->to_string()); + } else + // FIXME: require SRI hash. + input->hash = Hash(*hash); + } return input; } From b2c8061b44b01a39d953d75db30e39bd8f300021 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 9 Jun 2020 16:53:53 -0500 Subject: [PATCH 320/350] Disable extra-platforms = i686-linux on wsl1 (#3676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WSL1 doesn’t support i686-linux emulation, see https://github.com/microsoft/wsl/issues/2468 --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..2fbcafff8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -271,7 +271,7 @@ public: "listed in 'trusted-public-keys'."}; Setting extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", "Additional platforms that can be built on the local system. " "These may be supported natively (e.g. armv7 on some aarch64 CPUs " From 7eca8a16eaf74bc15a816e24005a65e5480d2a79 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 9 Jun 2020 23:35:38 -0500 Subject: [PATCH 321/350] Prelink static libraries into an object file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This combines the *.o into a big .o producing one translation unit. This preserve our unused static initializers, as specified in the C++ standard: If no variable or function is odr-used from a given translation unit, the non-local variables defined in that translation unit may never be initialized (this models the behavior of an on-demand dynamic library). Note that this is very similar to how the --whole-archive flag works. One advantage of this is that users of the final .a library don’t have to worry about specifying --whole-archive, or that we have unused static initializers at all! --- mk/libraries.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mk/libraries.mk b/mk/libraries.mk index 307e29b9d..e6ef2e3ec 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,8 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) $(AR) crs $$@ $$? + $(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + $(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) From 4750d98bbd7c233e993b1ed66dd5573e6bdf543c Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Mon, 8 Jun 2020 17:39:46 +0200 Subject: [PATCH 322/350] Add tests for pool.hh --- src/libutil/tests/pool.cc | 127 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 src/libutil/tests/pool.cc diff --git a/src/libutil/tests/pool.cc b/src/libutil/tests/pool.cc new file mode 100644 index 000000000..127e42dda --- /dev/null +++ b/src/libutil/tests/pool.cc @@ -0,0 +1,127 @@ +#include "pool.hh" +#include + +namespace nix { + + struct TestResource + { + + TestResource() { + static int counter = 0; + num = counter++; + } + + int dummyValue = 1; + bool good = true; + int num; + }; + + /* ---------------------------------------------------------------------------- + * Pool + * --------------------------------------------------------------------------*/ + + TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.capacity(), 1); + } + + TEST(Pool, freshPoolCanGetAResource) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.count(), 0); + + TestResource r = *(pool.get()); + + ASSERT_EQ(pool.count(), 1); + ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(r.dummyValue, 1); + ASSERT_EQ(r.good, true); + } + + TEST(Pool, capacityCanBeIncremented) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.incCapacity(); + ASSERT_EQ(pool.capacity(), 2); + } + + TEST(Pool, capacityCanBeDecremented) { + auto isGood = [](const ref & r) { return r->good; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.decCapacity(); + ASSERT_EQ(pool.capacity(), 0); + } + + TEST(Pool, flushBadDropsOutOfScopeResources) { + auto isGood = [](const ref & r) { return false; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + { + auto _r = pool.get(); + ASSERT_EQ(pool.count(), 1); + } + + pool.flushBad(); + ASSERT_EQ(pool.count(), 0); + } + + // Test that the resources we allocate are being reused when they are still good. + TEST(Pool, reuseResource) { + auto isGood = [](const ref & r) { return true; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them to be equal + // as the pool should hand out the same (still) good one again. + int counter = -1; + { + Pool::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { // the second handle should contain the same resource (with the same counter value) + Pool::Handle h = pool.get(); + ASSERT_EQ(h->num, counter); + } + } + + // Test that the resources we allocate are being thrown away when they are no longer good. + TEST(Pool, badResourceIsNotReused) { + auto isGood = [](const ref & r) { return false; }; + auto createResource = []() { return make_ref(); }; + + Pool pool = Pool((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them + // to *not* be equal as the pool should hand out a new instance after + // the first one was returned. + int counter = -1; + { + Pool::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { + // the second handle should contain a different resource (with a + //different counter value) + Pool::Handle h = pool.get(); + ASSERT_NE(h->num, counter); + } + } +} From 8bd892117a324c22803366b872bfe7ed783b34eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jun 2020 15:39:08 +0200 Subject: [PATCH 323/350] Style fixes --- src/libmain/loggers.cc | 8 ++++---- src/libmain/loggers.hh | 4 ++-- src/libmain/progress-bar.cc | 2 +- src/libmain/progress-bar.hh | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 41f0b35fb..350f90266 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -5,7 +5,7 @@ namespace nix { LogFormat defaultLogFormat = LogFormat::raw; -LogFormat parseLogFormat(const string &logFormatStr) { +LogFormat parseLogFormat(const std::string & logFormatStr) { if (logFormatStr == "raw") return LogFormat::raw; else if (logFormatStr == "raw-with-logs") @@ -19,7 +19,7 @@ LogFormat parseLogFormat(const string &logFormatStr) { throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); } -Logger *makeDefaultLogger() { +Logger * makeDefaultLogger() { switch (defaultLogFormat) { case LogFormat::raw: return makeSimpleLogger(false); @@ -34,11 +34,11 @@ Logger *makeDefaultLogger() { } } -void setLogFormat(const string &logFormatStr) { +void setLogFormat(const std::string & logFormatStr) { setLogFormat(parseLogFormat(logFormatStr)); } -void setLogFormat(const LogFormat &logFormat) { +void setLogFormat(const LogFormat & logFormat) { defaultLogFormat = logFormat; createDefaultLogger(); } diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f9ba5ee5e..cada03110 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -12,8 +12,8 @@ enum class LogFormat { barWithLogs, }; -void setLogFormat(const string &logFormatStr); -void setLogFormat(const LogFormat &logFormat); +void setLogFormat(const std::string & logFormatStr); +void setLogFormat(const LogFormat & logFormat); void createDefaultLogger(); diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index c9ba89714..b287de8a3 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -461,7 +461,7 @@ public: } }; -Logger *makeProgressBar(bool printBuildLogs) +Logger * makeProgressBar(bool printBuildLogs) { return new ProgressBar( printBuildLogs, diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 60d0a2076..7f0dafecf 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,7 +4,7 @@ namespace nix { -Logger* makeProgressBar(bool printBuildLogs = false); +Logger * makeProgressBar(bool printBuildLogs = false); void startProgressBar(bool printBuildLogs = false); From 95eb0640620f0704b4ded6ed6c7509024c00273e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jun 2020 15:39:30 +0200 Subject: [PATCH 324/350] Shut up warning --- src/libmain/loggers.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 350f90266..c44bb6408 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -31,6 +31,8 @@ Logger * makeDefaultLogger() { return makeProgressBar(); case LogFormat::barWithLogs: return makeProgressBar(true); + default: + abort(); } } From dd9bb11d0d38139bb32411170403171c4c92f8cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jun 2020 15:42:18 +0200 Subject: [PATCH 325/350] Move names.{cc,hh} to libstore --- src/{libutil => libstore}/names.cc | 0 src/{libutil => libstore}/names.hh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/{libutil => libstore}/names.cc (100%) rename src/{libutil => libstore}/names.hh (100%) diff --git a/src/libutil/names.cc b/src/libstore/names.cc similarity index 100% rename from src/libutil/names.cc rename to src/libstore/names.cc diff --git a/src/libutil/names.hh b/src/libstore/names.hh similarity index 100% rename from src/libutil/names.hh rename to src/libstore/names.hh From 045b07200c77bf1fe19c0a986aafb531e7e1ba54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jun 2020 12:46:33 +0200 Subject: [PATCH 326/350] Remove Store::queryDerivationOutputNames() This function was used in only one place, where it could easily be replaced by readDerivation() since it's not performance-critical. (This function appears to have been modelled after queryDerivationOutputs(), which exists only to make the garbage collector faster.) --- src/libexpr/primops.cc | 2 +- src/libstore/build.cc | 3 --- src/libstore/daemon.cc | 3 +-- src/libstore/derivations.cc | 9 +++++++++ src/libstore/derivations.hh | 2 ++ src/libstore/local-store.cc | 17 ----------------- src/libstore/local-store.hh | 2 -- src/libstore/remote-store.cc | 9 --------- src/libstore/remote-store.hh | 2 -- src/libstore/store-api.hh | 4 ---- src/libstore/worker-protocol.hh | 2 +- 11 files changed, 14 insertions(+), 41 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d458ab272..9efb8eae5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -688,7 +688,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : refs) { drv.inputSrcs.insert(j.clone()); if (j.isDerivation()) - drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j); + drv.inputDrvs[j.clone()] = readDerivation(*state.store, state.store->toRealPath(j)).outputNames(); } } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index bdf03ff94..3ab6220e3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2723,9 +2723,6 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const StorePath & path) override - { throw Error("queryDerivationOutputNames"); } - std::optional queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5cff170dd..dc2015579 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -329,8 +329,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); + auto names = readDerivation(*store, store->toRealPath(path)).outputNames(); logger->stopWork(); to << names; break; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b..6de91ec97 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -410,6 +410,15 @@ StorePathSet BasicDerivation::outputPaths() const } +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93b..88aed66bf 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -58,6 +58,8 @@ struct BasicDerivation /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1c3795eb1..e379db426 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -785,23 +785,6 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) } -StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) -{ - return retrySQLite([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..e17cc45ae 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -135,8 +135,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..735f59a91 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << printStorePath(path); - conn.processStderr(); - return readStrings(conn->from); -} - - std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..80c8e9f11 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,8 +51,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..5ba17e0bc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -430,10 +430,6 @@ public: virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } - /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 857d54d99..ac42457fc 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, From 9f736dd89d064718a378703a9557de56fdd1a734 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jun 2020 13:04:52 +0200 Subject: [PATCH 327/350] Add Store::readDerivation() convenience function --- src/build-remote/build-remote.cc | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/derivations.cc | 8 +++++++- src/libstore/local-store.cc | 5 ++--- src/libstore/store-api.hh | 3 +++ src/nix/show-derivation.cc | 6 ++---- 7 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 00340b787..d2ea6c956 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -241,7 +241,7 @@ connected: uploadLock = -1; - BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string()))); + auto drv = store->readDerivation(*drvPath); drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9efb8eae5..3a27527a0 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -688,7 +688,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : refs) { drv.inputSrcs.insert(j.clone()); if (j.isDerivation()) - drv.inputDrvs[j.clone()] = readDerivation(*state.store, state.store->toRealPath(j)).outputNames(); + drv.inputDrvs[j.clone()] = state.store->readDerivation(j).outputNames(); } } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index dc2015579..877c70a03 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -329,7 +329,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - auto names = readDerivation(*store, store->toRealPath(path)).outputNames(); + auto names = store->readDerivation(path).outputNames(); logger->stopWork(); to << names; break; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6de91ec97..e268c65ff 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -204,6 +204,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); @@ -378,7 +384,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(i.first)), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e379db426..c2c45b4c2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -595,7 +595,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -911,8 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5ba17e0bc..5ef506326 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -583,6 +583,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 22c569f3c..2d31894c2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -61,11 +61,9 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - auto drvPathS = store->printStorePath(drvPath); + auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); - auto drvObj(jsonRoot.object(drvPathS)); - - auto drv = readDerivation(*store, drvPathS); + auto drv = store->readDerivation(drvPath); { auto outputsObj(drvObj.object("outputs")); From b260c9ee0325d7a4c78d91e47bd6e2afbde16e28 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 10:09:42 -0500 Subject: [PATCH 328/350] Add newHashAllowEmpty helper function This replaces the copy&paste with a helper function in hash.hh. --- src/libexpr/primops.cc | 18 ++++-------------- src/libexpr/primops/fetchTree.cc | 11 +++-------- src/libfetchers/fetchers.cc | 11 +++-------- src/libfetchers/tarball.cc | 10 ++-------- src/libutil/hash.cc | 10 ++++++++++ src/libutil/hash.hh | 2 ++ 6 files changed, 24 insertions(+), 38 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index df3d4a459..083c7d398 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -720,12 +720,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h; - if (outputHash->empty()) { - h = Hash(ht); - printError("warning: found empty hash, assuming you wanted '%s'", h.to_string()); - } else - h = Hash(*outputHash, ht); + Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); @@ -1131,14 +1126,9 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value filterFun = attr.value; } else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; - else if (n == "sha256") { - auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos); - if (hashStr == "") { - expectedHash = Hash(htSHA256); - printError("warning: found empty hash, assuming you wanted '%s'", expectedHash.to_string()); - } else - expectedHash = Hash(hashStr, htSHA256); - } else + else if (n == "sha256") + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); } if (path.empty()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 745f65adf..1464aa8b4 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -102,14 +102,9 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, string n(attr.name); if (n == "url") url = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") { - auto hashStr = state.forceStringNoCtx(*attr.value, *attr.pos); - if (hashStr == "") { - expectedHash = Hash(htSHA256); - printError("warning: found empty hash, assuming you wanted '%s'", expectedHash->to_string()); - } else - expectedHash = Hash(hashStr, htSHA256); - } else if (n == "name") + else if (n == "sha256") + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 988a8dc69..91b897e62 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -34,14 +34,9 @@ std::unique_ptr inputFromAttrs(const Attrs & attrs) for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromAttrs(attrs2); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) { - if (narHash->empty()) { - res->narHash = Hash(htUnknown); - printError("warning: found empty hash, assuming you wanted '%s'", res->narHash->to_string()); - } else - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - } + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = newHashAllowEmpty(*narHash, htUnknown); return res; } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e4dafec0b..937f86bc6 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -263,14 +263,8 @@ struct TarballInputScheme : InputScheme throw Error("unsupported tarball input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) { - if (hash->empty()) { - input->hash = Hash(htUnknown); - printError("warning: found empty hash, assuming you wanted '%s'", input->hash->to_string()); - } else - // FIXME: require SRI hash. - input->hash = Hash(*hash); - } + if (auto hash = maybeGetStrAttr(attrs, "hash")) + input->hash = newHashAllowEmpty(*hash, htUnknown); return input; } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7caee1da7..2c8085e42 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -205,6 +205,16 @@ Hash::Hash(const std::string & s, HashType type) throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } +Hash newHashAllowEmpty(std::string hashStr, HashType ht) +{ + if (hashStr.empty()) + { + Hash h(ht); + warn("found empty hash, assuming you wanted '%s'", h.to_string()); + } else + return Hash(hashStr, ht); +} + union Ctx { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ea9fca3e7..449cb1b86 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -94,6 +94,8 @@ struct Hash } }; +/* Helper that defaults empty hashes to the 0 hash. */ +Hash newHashAllowEmpty(std::string hashStr, HashType ht); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ string printHash16or32(const Hash & hash); From ea0d29d99a400c328fa0ca05ba5e639351673ebc Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 10:18:27 -0500 Subject: [PATCH 329/350] Provide base argument to to_string --- src/libutil/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c8085e42..3f6d4c0c9 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -210,7 +210,7 @@ Hash newHashAllowEmpty(std::string hashStr, HashType ht) if (hashStr.empty()) { Hash h(ht); - warn("found empty hash, assuming you wanted '%s'", h.to_string()); + warn("found empty hash, assuming you wanted '%s'", h.to_string(SRI)); } else return Hash(hashStr, ht); } From 2853ba4ab26c254d564aee9e75fe8f9f664b94fc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jun 2020 19:00:48 +0200 Subject: [PATCH 330/350] Fix build --- src/libutil/hash.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 3f6d4c0c9..c935f05eb 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -207,10 +207,10 @@ Hash::Hash(const std::string & s, HashType type) Hash newHashAllowEmpty(std::string hashStr, HashType ht) { - if (hashStr.empty()) - { + if (hashStr.empty()) { Hash h(ht); - warn("found empty hash, assuming you wanted '%s'", h.to_string(SRI)); + warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + return h; } else return Hash(hashStr, ht); } From f6f01416b71f239c0d9594f5c6201dfd208fe8c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 12 Jun 2020 21:12:36 +0000 Subject: [PATCH 331/350] Use `std::string_view` in a few more places --- src/libfetchers/github.cc | 2 +- src/libstore/store-api.cc | 4 ++-- src/libutil/hash.cc | 4 ++-- src/libutil/hash.hh | 2 +- src/libutil/util.cc | 6 +++--- src/libutil/util.hh | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8675a5a66..0bee1d6b3 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -76,7 +76,7 @@ struct GitHubInput : Input readFile( store->toRealPath( downloadFile(store, url, "source", false).storePath))); - rev = Hash(json["sha"], htSHA1); + rev = Hash(std::string { json["sha"] }, htSHA1); debug("HEAD revision for '%s' is %s", url, rev->gitRev()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0645fca84..f6901bf42 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -779,7 +779,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const }; if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); + Hash hash(ca.substr(5)); if (store.makeTextPath(path.name(), hash, references) == path) return true; else @@ -788,7 +788,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index c935f05eb..460d479a3 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -125,7 +125,7 @@ std::string Hash::to_string(Base base, bool includeType) const } -Hash::Hash(const std::string & s, HashType type) +Hash::Hash(std::string_view s, HashType type) : type(type) { size_t pos = 0; @@ -194,7 +194,7 @@ Hash::Hash(const std::string & s, HashType type) } else if (isSRI || size == base64Len()) { - auto d = base64Decode(std::string(s, pos)); + auto d = base64Decode(s.substr(pos)); if (d.size() != hashSize) throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); assert(hashSize); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 5080539b2..180fb7633 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -42,7 +42,7 @@ struct Hash Subresource Integrity hash expression). If the 'type' argument is htUnknown, then the hash type must be specified in the string. */ - Hash(const std::string & s, HashType type = htUnknown); + Hash(std::string_view s, HashType type = htUnknown); void init(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e0a99152b..b66447e08 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1314,7 +1314,7 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & prefix) +bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; } @@ -1408,7 +1408,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -string base64Encode(const string & s) +string base64Encode(std::string_view s) { string res; int data = 0, nbits = 0; @@ -1429,7 +1429,7 @@ string base64Encode(const string & s) } -string base64Decode(const string & s) +string base64Decode(std::string_view s) { bool init = false; char decode[256]; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 4b117f9bc..c95232317 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -416,7 +416,7 @@ template bool string2Float(const string & s, N & n) /* Return true iff `s' starts with `prefix'. */ -bool hasPrefix(const string & s, const string & prefix); +bool hasPrefix(std::string_view s, std::string_view prefix); /* Return true iff `s' ends in `suffix'. */ @@ -455,8 +455,8 @@ std::string filterANSIEscapes(const std::string & s, /* Base64 encoding/decoding. */ -string base64Encode(const string & s); -string base64Decode(const string & s); +string base64Encode(std::string_view s); +string base64Decode(std::string_view s); /* Get a value for the specified key from an associate container. */ From 340d0b055afb02ec7292d6cd9dd1661640158210 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Jun 2020 14:57:24 +0200 Subject: [PATCH 332/350] upload-release.pl: Fix nix-fallback-paths.nix generation --- maintainers/upload-release.pl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index cb584d427..baefe0f12 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -142,8 +142,12 @@ $oldName =~ s/"//g; sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build"; - return $buildInfo->{buildproducts}->{1}->{path}; + for my $product (values %{$buildInfo->{buildproducts}}) { + next unless $product->{type} eq "nix-build"; + next if $product->{path} =~ /[a-z]+$/; + return $product->{path}; + } + die; } write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", From fd64e4fb96f814440dc337ce664cdbd22e0eabb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 13:50:33 +0200 Subject: [PATCH 333/350] Disambiguate BaseError(Args) constructor This means that 'throw Error({ ... ErrorInfo ... })' now works. --- src/libexpr/eval.cc | 9 ++++----- src/libutil/types.hh | 3 --- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7bf25ea17..75e6a23a1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -506,11 +506,10 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { - throw EvalError( - ErrorInfo { - .hint = hintfmt(s, s2), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt(s, s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 89ae108f9..3af485fa0 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,6 +1,5 @@ #pragma once - #include "ref.hh" #include @@ -25,7 +24,6 @@ typedef string Path; typedef list Paths; typedef set PathSet; - /* Helper class to run code at startup. */ template struct OnStartup @@ -33,5 +31,4 @@ struct OnStartup OnStartup(T && t) { t(); } }; - } From 1fb762d11fadc659ef41b79eaddddcce5fbbc192 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 14:06:58 +0200 Subject: [PATCH 334/350] Get rid of explicit ErrorInfo constructors --- src/libexpr/attr-set.hh | 9 +- src/libexpr/eval-inline.hh | 18 +- src/libexpr/eval.cc | 65 ++-- src/libexpr/nixexpr.cc | 9 +- src/libexpr/nixexpr.hh | 9 +- src/libexpr/parser.y | 86 +++-- src/libexpr/primops.cc | 442 ++++++++++++-------------- src/libexpr/primops/context.cc | 27 +- src/libexpr/primops/fetchGit.cc | 18 +- src/libexpr/primops/fetchMercurial.cc | 18 +- src/libexpr/primops/fetchTree.cc | 28 +- src/libexpr/primops/fromTOML.cc | 9 +- src/libstore/builtins/buildenv.cc | 14 +- src/libstore/gc.cc | 11 +- src/libstore/sqlite.cc | 8 +- src/libutil/error.hh | 14 +- src/libutil/serialise.cc | 4 +- src/nix-build/nix-build.cc | 11 +- src/nix/upgrade-nix.cc | 7 +- 19 files changed, 360 insertions(+), 447 deletions(-) diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index f5651891f..c601d09c2 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -76,11 +76,10 @@ public: { auto a = get(name); if (!a) - throw Error( - ErrorInfo { - .hint = hintfmt("attribute '%s' missing", name), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("attribute '%s' missing", name), + .nixCode = NixCode { .errPos = pos } + }); return *a; } diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index eee49e02e..3d544c903 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -9,11 +9,10 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { - throw EvalError( - ErrorInfo { - .hint = hintfmt(s), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) @@ -24,11 +23,10 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { - throw TypeError( - ErrorInfo { - .hint = hintfmt(s, showType(v)), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt(s, showType(v)), + .nixCode = NixCode { .errPos = pos } + }); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 75e6a23a1..8e71db2b8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -519,30 +519,27 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { - throw EvalError( - ErrorInfo { - .hint = hintfmt(s, s2, s3), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt(s, s2, s3), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2)) { // p1 is where the error occurred; p2 is a position mentioned in the message. - throw EvalError( - ErrorInfo { - .hint = hintfmt(s, sym, p2), - .nixCode = NixCode { .errPos = p1 } - }); + throw EvalError({ + .hint = hintfmt(s, sym, p2), + .nixCode = NixCode { .errPos = p1 } + }); } LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { - throw TypeError( - ErrorInfo { - .hint = hintfmt(s), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) @@ -552,29 +549,26 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { - throw TypeError( - ErrorInfo { - .hint = hintfmt(s, fun.showNamePos(), s2), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt(s, fun.showNamePos(), s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { - throw AssertionError( - ErrorInfo { - .hint = hintfmt(s, s1), - .nixCode = NixCode { .errPos = pos } - }); + throw AssertionError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { - throw UndefinedVarError( - ErrorInfo { - .hint = hintfmt(s, s1), - .nixCode = NixCode { .errPos = pos } - }); + throw UndefinedVarError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) @@ -1593,7 +1587,7 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) string s = forceString(v, pos); if (v.string.context) { if (pos) - throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]); else throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", @@ -1919,11 +1913,10 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { - throw TypeError( - ErrorInfo { - .hint = hintfmt("cannot coerce %1% to a string", showType()), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt("cannot coerce %1% to a string", showType()), + .nixCode = NixCode { .errPos = pos } + }); } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 91a508305..5b2dd9751 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -268,11 +268,10 @@ void ExprVar::bindVars(const StaticEnv & env) enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ if (withLevel == -1) - throw UndefinedVarError( - ErrorInfo { - .hint = hintfmt("undefined variable '%1%'", name), - .nixCode = NixCode { .errPos = pos } - }); + throw UndefinedVarError({ + .hint = hintfmt("undefined variable '%1%'", name), + .nixCode = NixCode { .errPos = pos } + }); fromWith = true; this->level = withLevel; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 47d0e85ec..ec6fd3190 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -236,11 +236,10 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError( - ErrorInfo { - .hint = hintfmt("duplicate formal function argument '%1%'", arg), - .nixCode = NixCode { .errPos = pos } - }); + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .nixCode = NixCode { .errPos = pos } + }); }; void setName(Symbol & name); string showNamePos() const; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0417a3c21..a1188dec0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,23 +64,20 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { - throw ParseError( - ErrorInfo { - .hint = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(attrPath), prevPos), - .nixCode = NixCode { .errPos = pos }, - }); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { - throw ParseError( - ErrorInfo { - .hint = hintfmt("attribute '%1%' already defined at %2%", - attr, prevPos), - .nixCode = NixCode { .errPos = pos }, - }); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } @@ -148,12 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) - throw ParseError( - ErrorInfo { - .hint = hintfmt("duplicate formal function argument '%1%'", - formal.name), - .nixCode = NixCode { .errPos = pos }, - }); + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", + formal.name), + .nixCode = NixCode { .errPos = pos }, + }); formals->formals.push_front(formal); } @@ -261,10 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { - data->error = ErrorInfo { + data->error = { .hint = hintfmt(error), .nixCode = NixCode { .errPos = makeCurPos(*loc, data) } - }; + }; } @@ -341,11 +337,10 @@ expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) - throw ParseError( - ErrorInfo { - .hint = hintfmt("dynamic attributes not allowed in let"), - .nixCode = NixCode { .errPos = CUR_POS }, - }); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in let"), + .nixCode = NixCode { .errPos = CUR_POS }, + }); $$ = new ExprLet($2, $4); } | expr_if @@ -422,11 +417,10 @@ expr_simple | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) - throw ParseError( - ErrorInfo { - .hint = hintfmt("URL literals are disabled"), - .nixCode = NixCode { .errPos = CUR_POS } - }); + throw ParseError({ + .hint = hintfmt("URL literals are disabled"), + .nixCode = NixCode { .errPos = CUR_POS } + }); $$ = new ExprString(data->symbols.create($1)); } | '(' expr ')' { $$ = $2; } @@ -496,11 +490,10 @@ attrs $$->push_back(AttrName(str->s)); delete str; } else - throw ParseError( - ErrorInfo { - .hint = hintfmt("dynamic attributes not allowed in inherit"), - .nixCode = NixCode { .errPos = makeCurPos(@2, data) }, - }); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in inherit"), + .nixCode = NixCode { .errPos = makeCurPos(@2, data) }, + }); } | { $$ = new AttrPath; } ; @@ -695,11 +688,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - throw ThrownError( - ErrorInfo { - .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), - .nixCode = NixCode { .errPos = pos } - }); + throw ThrownError({ + .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), + .nixCode = NixCode { .errPos = pos } + }); } @@ -715,10 +707,9 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl res = { true, store->toRealPath(fetchers::downloadTarball( store, resolveUri(elem.second), "source", false).storePath) }; } catch (FileTransferError & e) { - logWarning( - ErrorInfo { - .name = "Entry download", - .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + logWarning({ + .name = "Entry download", + .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } @@ -727,10 +718,9 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (pathExists(path)) res = { true, path }; else { - logWarning( - ErrorInfo { - .name = "Entry not found", - .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + logWarning({ + .name = "Entry not found", + .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 23ab7dec6..62e5163c9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -93,12 +93,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", - path, e.path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -174,13 +172,12 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt( - "cannot import '%1%', since path '%2%' is not valid", - path, e.path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt( + "cannot import '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } path = state.checkSourcePath(path); @@ -215,11 +212,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("at least one argument to 'exec' required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("at least one argument to 'exec' required"), + .nixCode = NixCode { .errPos = pos } + }); } PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); @@ -230,12 +226,12 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", - program, e.path), - .nixCode = NixCode { .errPos = pos } - });} + throw EvalError({ + .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + program, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } auto output = runProgram(program, true, commandArgs); Expr * parsed; @@ -386,11 +382,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator startSet = args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("attribute 'startSet' required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("attribute 'startSet' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceList(*startSet->value, pos); ValueList workSet; @@ -401,11 +396,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator op = args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("attribute 'operator' required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("attribute 'operator' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of @@ -424,11 +418,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator key = e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("attribute 'key' required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("attribute 'key' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*key->value, pos); if (!doneKeys.insert(key->value).second) continue; @@ -560,11 +553,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Figure out the name first (for stack backtraces). */ Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("required attribute 'name' missing"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("required attribute 'name' missing"), + .nixCode = NixCode { .errPos = pos } + }); string drvName; Pos & posDrvName(*attr->pos); try { @@ -607,42 +599,38 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto handleHashMode = [&](const std::string & s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .nixCode = NixCode { .errPos = posDrvName } - }); + else + throw EvalError({ + .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .nixCode = NixCode { .errPos = posDrvName } + }); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("duplicate derivation output '%1%'", j), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("duplicate derivation output '%1%'", j), + .nixCode = NixCode { .errPos = posDrvName } + }); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid derivation output name 'drv'" ), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("invalid derivation output name 'drv'" ), + .nixCode = NixCode { .errPos = posDrvName } + }); outputs.insert(j); } if (outputs.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("derivation cannot have an empty set of outputs"), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("derivation cannot have an empty set of outputs"), + .nixCode = NixCode { .errPos = posDrvName } + }); }; try { @@ -754,35 +742,31 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError( - ErrorInfo { - .hint = hintfmt("required attribute 'builder' missing"), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("required attribute 'builder' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); if (drv.platform == "") - throw EvalError( - ErrorInfo { - .hint = hintfmt("required attribute 'system' missing"), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("required attribute 'system' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) - throw EvalError( - ErrorInfo { - .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw EvalError({ + .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .nixCode = NixCode { .errPos = posDrvName } + }); if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error( - ErrorInfo { - .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .nixCode = NixCode { .errPos = posDrvName } - }); + throw Error({ + .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .nixCode = NixCode { .errPos = posDrvName } + }); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); @@ -887,11 +871,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) - throw EvalError( - ErrorInfo { - .hint = hintfmt("path '%1%' is not in the Nix store", path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("path '%1%' is not in the Nix store", path), + .nixCode = NixCode { .errPos = pos } + }); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(path2)); @@ -907,13 +890,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt( - "cannot check the existence of '%1%', since path '%2%' is not valid", - path, e.path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt( + "cannot check the existence of '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } try { @@ -956,13 +938,11 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid" - , path, e.path), - .nixCode = NixCode { .errPos = pos } - }); - } + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); + } string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); @@ -989,11 +969,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("attribute 'path' missing"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("attribute 'path' missing"), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -1001,12 +980,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", - path, e.path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } searchPath.emplace_back(prefix, path); @@ -1023,11 +1000,10 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error( - ErrorInfo { - .hint = hintfmt("unknown hash type '%1%'", type), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded Path p = state.coerceToPath(pos, *args[1], context); @@ -1043,12 +1019,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val try { state.realiseContext(ctx); } catch (InvalidPathError & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", - path, e.path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } DirEntries entries = readDirectory(state.checkSourcePath(path)); @@ -1118,15 +1092,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError( - ErrorInfo { - .hint = hintfmt( - "in 'toFile': the file named '%1%' must not contain a reference " - "to a derivation but contains (%2%)", - name, - path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError( { + .hint = hintfmt( + "in 'toFile': the file named '%1%' must not contain a reference " + "to a derivation but contains (%2%)", + name, path), + .nixCode = NixCode { .errPos = pos } + }); refs.insert(state.store->parseStorePath(path)); } @@ -1194,21 +1166,19 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("string '%1%' cannot refer to other paths", path), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError( - ErrorInfo { - .hint = hintfmt( - "first argument in call to 'filterSource' is not a function but %1%", - showType(*args[0])), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt( + "first argument in call to 'filterSource' is not a function but %1%", + showType(*args[0])), + .nixCode = NixCode { .errPos = pos } + }); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); } @@ -1228,12 +1198,10 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value PathSet context; path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("string '%1%' cannot refer to other paths", - path), - .nixCode = NixCode { .errPos = *attr.pos } - }); + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = *attr.pos } + }); } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { @@ -1244,19 +1212,16 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else - throw EvalError( - ErrorInfo { - .hint = hintfmt("unsupported argument '%1%' to 'addPath'", - attr.name), - .nixCode = NixCode { .errPos = *attr.pos } - }); + throw EvalError({ + .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (path.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("'path' required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("'path' required"), + .nixCode = NixCode { .errPos = pos } + }); if (name.empty()) name = baseNameOf(path); @@ -1314,11 +1279,10 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("attribute '%1%' missing", attr), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("attribute '%1%' missing", attr), + .nixCode = NixCode { .errPos = pos } + }); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; state.forceValue(*i->value, pos); @@ -1398,22 +1362,20 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) - throw TypeError( - ErrorInfo { - .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) - throw TypeError( - ErrorInfo { - .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } } @@ -1486,11 +1448,10 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args { state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError( - ErrorInfo { - .hint = hintfmt("'functionArgs' requires a function"), - .nixCode = NixCode { .errPos = pos } - }); + throw TypeError({ + .hint = hintfmt("'functionArgs' requires a function"), + .nixCode = NixCode { .errPos = pos } + }); if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); @@ -1543,11 +1504,10 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error( - ErrorInfo { - .hint = hintfmt("list index %1% is out of bounds", n), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("list index %1% is out of bounds", n), + .nixCode = NixCode { .errPos = pos } + }); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -1574,11 +1534,10 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value { state.forceList(*args[0], pos); if (args[0]->listSize() == 0) - throw Error( - ErrorInfo { - .hint = hintfmt("'tail' called on an empty list"), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("'tail' called on an empty list"), + .nixCode = NixCode { .errPos = pos } + }); state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) @@ -1720,12 +1679,10 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val auto len = state.forceInt(*args[1], pos); if (len < 0) - throw EvalError( - ErrorInfo { - .hint = hintfmt("cannot create list of size %1%", len), - .nixCode = NixCode { .errPos = pos } - }); - + throw EvalError({ + .hint = hintfmt("cannot create list of size %1%", len), + .nixCode = NixCode { .errPos = pos } + }); state.mkList(v, len); @@ -1883,12 +1840,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[1], pos); NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) - throw EvalError( - ErrorInfo { - .hint = hintfmt("division by zero"), - .nixCode = NixCode { .errPos = pos } - }); + if (f2 == 0) + throw EvalError({ + .hint = hintfmt("division by zero"), + .nixCode = NixCode { .errPos = pos } + }); if (args[0]->type == tFloat || args[1]->type == tFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); @@ -1897,11 +1853,10 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixInt i2 = state.forceInt(*args[1], pos); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - throw EvalError( - ErrorInfo { - .hint = hintfmt("overflow in integer division"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("overflow in integer division"), + .nixCode = NixCode { .errPos = pos } + }); mkInt(v, i1 / i2); } @@ -1958,12 +1913,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V PathSet context; string s = state.coerceToString(pos, *args[2], context); - if (start < 0) - throw EvalError( - ErrorInfo { - .hint = hintfmt("negative start position in 'substring'"), - .nixCode = NixCode { .errPos = pos } - }); + if (start < 0) + throw EvalError({ + .hint = hintfmt("negative start position in 'substring'"), + .nixCode = NixCode { .errPos = pos } + }); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } @@ -1983,11 +1937,10 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error( - ErrorInfo { - .hint = hintfmt("unknown hash type '%1%'", type), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); @@ -2030,17 +1983,15 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError( - ErrorInfo { - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } else { - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -2105,17 +2056,15 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError( - ErrorInfo { - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } else { - throw EvalError( - ErrorInfo { - .hint = hintfmt("invalid regular expression '%s'", re), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -2146,11 +2095,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[0], pos); state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .nixCode = NixCode { .errPos = pos } + }); vector from; from.reserve(args[0]->listSize()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 7f895fc01..efa2e9576 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -146,11 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg auto sAllOutputs = state.symbols.create("allOutputs"); for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) - throw EvalError( - ErrorInfo { - .hint = hintfmt("Context key '%s' is not a store path", i.name), - .nixCode = NixCode { .errPos = *i.pos } - }); + throw EvalError({ + .hint = hintfmt("Context key '%s' is not a store path", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(i.name)); state.forceAttrs(*i.value, *i.pos); @@ -164,11 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), - .nixCode = NixCode { .errPos = *i.pos } - }); + throw EvalError({ + .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); } context.insert("=" + string(i.name)); } @@ -178,11 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), - .nixCode = NixCode { .errPos = *i.pos } - }); + throw EvalError({ + .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); } for (unsigned int n = 0; n < iter->value->listSize(); ++n) { auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 52826b56c..dd7229a3d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -35,19 +35,17 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "submodules") fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else - throw EvalError( - ErrorInfo { - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .nixCode = NixCode { .errPos = *attr.pos } - }); + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (url.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("'url' argument required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index bb008ba6b..9bace8f89 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -38,19 +38,17 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError( - ErrorInfo { - .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), - .nixCode = NixCode { .errPos = *attr.pos } - }); + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (url.empty()) - throw EvalError( - ErrorInfo { - .hint = hintfmt("'url' argument required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 62bc4f433..9be93710a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -66,11 +66,10 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V } if (!attrs.count("type")) - throw Error( - ErrorInfo { - .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .nixCode = NixCode { .errPos = pos } - }); + throw Error({ + .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .nixCode = NixCode { .errPos = pos } + }); input = fetchers::inputFromAttrs(attrs); } else @@ -111,20 +110,17 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError( - ErrorInfo { - .hint = hintfmt("unsupported argument '%s' to '%s'", - attr.name, who), - .nixCode = NixCode { .errPos = *attr.pos } - }); + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (!url) - throw EvalError( - ErrorInfo { - .hint = hintfmt("'url' argument required"), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.forceStringNoCtx(*args[0], pos); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 948069401..7615d1379 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -81,11 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va try { visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { - throw EvalError( - ErrorInfo { - .hint = hintfmt("while parsing a TOML string: %s", e.what()), - .nixCode = NixCode { .errPos = pos } - }); + throw EvalError({ + .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .nixCode = NixCode { .errPos = pos } + }); } } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 6c493ed77..802fb87bc 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,10 +22,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - logWarning( - ErrorInfo { - .name = "Create links - directory", - .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + logWarning({ + .name = "Create links - directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) }); return; } @@ -45,10 +44,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - logWarning( - ErrorInfo { - .name = "Create links - skipping symlink", - .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + logWarning({ + .name = "Create links - skipping symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) }); continue; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 92f9328c3..04e3849f7 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -129,12 +129,11 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, if (settings.checkRootReachability) { auto roots = findRoots(false); if (roots[storePath.clone()].count(gcRoot) == 0) - logWarning( - ErrorInfo { - .name = "GC root", - .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)) + logWarning({ + .name = "GC root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)) }); } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 16cdb6619..76c822c4e 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -204,10 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; - logWarning( - ErrorInfo { .name = "Sqlite busy", - .hint = hintfmt(e.what()) - }); + logWarning({ + .name = "Sqlite busy", + .hint = hintfmt(e.what()) + }); } /* Sleep for a while since retrying the transaction right away diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 8a48fa105..2ba3a3b33 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -22,7 +22,7 @@ namespace nix { -/* +/* This file defines two main structs/classes used in nix error handling. @@ -106,7 +106,7 @@ protected: mutable std::optional what_; const string& calcWhat() const; - + public: unsigned int status = 1; // exit status @@ -119,9 +119,9 @@ public: { } template - BaseError(const Args & ... args) + BaseError(const std::string & fs, const Args & ... args) : err { .level = lvlError, - .hint = hintfmt(args...) + .hint = hintfmt(fs, args...) } { } @@ -131,7 +131,11 @@ public: } { } - BaseError(ErrorInfo e) + BaseError(ErrorInfo && e) + : err(std::move(e)) + { } + + BaseError(const ErrorInfo & e) : err(e) { } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 35f7ee917..c8b71188f 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -52,10 +52,10 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - logWarning(ErrorInfo { + logWarning({ .name = "Large path", .description = "dumping very large path (> 256 MiB); this may run out of memory" - }); + }); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 591fff999..da6f005a3 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -368,12 +368,11 @@ static void _main(int argc, char * * argv) shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - logWarning( - ErrorInfo { - .name = "bashInteractive", - .hint = hintfmt("%s; will use bash from your environment", - (e.info().hint ? e.info().hint->str() : "")) - }); + logWarning({ + .name = "bashInteractive", + .hint = hintfmt("%s; will use bash from your environment", + (e.info().hint ? e.info().hint->str() : "")) + }); shell = "bash"; } } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index fdf94e5a3..a880bdae0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -68,10 +68,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - logWarning( - ErrorInfo { - .name = "Version update", - .hint = hintfmt("would upgrade to version %s", version) + logWarning({ + .name = "Version update", + .hint = hintfmt("would upgrade to version %s", version) }); return; } From e14e62fddde3b00ee82a8da29725f571ad8ecee1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 14:12:39 +0200 Subject: [PATCH 335/350] Remove trailing whitespace --- src/build-remote/build-remote.cc | 2 +- src/libexpr/attr-path.cc | 4 +-- src/libexpr/lexer.l | 1 - src/libexpr/nixexpr.cc | 2 +- src/libmain/progress-bar.cc | 2 +- src/libstore/daemon.cc | 2 +- src/libstore/filetransfer.cc | 4 +-- src/libstore/local-store.cc | 10 +++---- src/libstore/optimise-store.cc | 6 ++-- src/libstore/store-api.cc | 2 +- src/libutil/error.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/util.cc | 2 +- src/nix-daemon/nix-daemon.cc | 30 +++++++++---------- src/nix-env/nix-env.cc | 8 ++--- src/nix-store/nix-store.cc | 4 +-- src/nix/verify.cc | 6 ++-- .../resolve-system-dependencies.cc | 8 ++--- 18 files changed, 48 insertions(+), 49 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 49b35a4e9..e07117496 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -200,7 +200,7 @@ static int _main(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - logError({ + logError({ .name = "Remote build", .hint = hintfmt("cannot build on '%s': %s%s", bestMachine->storeUri, e.what(), diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9a9531a3f..8980bc09d 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -62,7 +62,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, - showType(*v)); + showType(*v)); if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); @@ -79,7 +79,7 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, - showType(*v)); + showType(*v)); if (attrIndex >= v->listSize()) throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 85376a08f..f6e83926b 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -219,4 +219,3 @@ or { return OR_KW; } } %% - diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 5b2dd9751..b4b65883d 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -267,7 +267,7 @@ void ExprVar::bindVars(const StaticEnv & env) /* Otherwise, the variable must be obtained from the nearest enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ - if (withLevel == -1) + if (withLevel == -1) throw UndefinedVarError({ .hint = hintfmt("undefined variable '%1%'", name), .nixCode = NixCode { .errPos = pos } diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 20d9915a0..8d0421c2f 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -133,7 +133,7 @@ public: { auto state(state_.lock()); - std::stringstream oss; + std::stringstream oss; oss << ei; log(*state, ei.level, oss.str()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 172cfe3cb..620722516 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -77,7 +77,7 @@ struct TunnelLogger : public Logger { if (ei.level > verbosity) return; - std::stringstream oss; + std::stringstream oss; oss << ei; StringSink buf; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 6ca3393ab..601cb858a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -599,9 +599,9 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - logError({ + logError({ .name = "File transfer", - .hint = hintfmt("unexpected error in download thread: %s", + .hint = hintfmt("unexpected error in download thread: %s", e.what()) }); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5521b8633..f5c5bd9b7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,7 +87,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - logError({ + logError({ .name = "'build-users-group' not found", .hint = hintfmt( "warning: the group '%1%' specified in 'build-users-group' does not exist", @@ -1223,7 +1223,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - logError({ + logError({ .name = "Invalid hash", .hint = hintfmt( "link '%s' was modified! expected hash '%s', got '%s'", @@ -1261,7 +1261,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - logError({ + logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) @@ -1316,7 +1316,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - logError({ + logError({ .name = "Nix path not found", .hint = hintfmt("path '%s' is not in the Nix store", pathS) }); @@ -1342,7 +1342,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - logError({ + logError({ .name = "Missing path with referrers", .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) }); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index d760d110c..b2b2412a3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -130,7 +130,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - logWarning({ + logWarning({ .name = "Suspicious file", .hint = hintfmt("skipping suspicious writable file '%1%'", path) }); @@ -197,7 +197,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - logWarning({ + logWarning({ .name = "Corrupted link", .hint = hintfmt("removing corrupted link '%1%'", linkPath) }); @@ -235,7 +235,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - logError({ + logError({ .name = "Unlink error", .hint = hintfmt("unable to unlink '%1%'", tempLink) }); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 464261010..e23a9ca50 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -777,7 +777,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const auto warn = [&]() { logWarning( ErrorInfo{ - .name = "Path not content-addressed", + .name = "Path not content-addressed", .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) }); }; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1fcb8111c..cb5fe70bf 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -61,7 +61,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC { // previous line of code. if (nixCode.prevLineOfCode.has_value()) { - out << std::endl + out << std::endl << fmt("%1% %|2$5d|| %3%", prefix, (nixCode.errPos.line - 1), diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 108dc3bd1..105fadb15 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -273,7 +273,7 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - logError({ + logError({ .name = "Json log message", .hint = hintfmt("bad log message from builder: %s", e.what()) }); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a2281237b..667dd2edb 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -314,7 +314,7 @@ string readFile(const Path & path) void readFile(const Path & path, Sink & sink) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) + if (!fd) throw SysError("opening file '%s'", path); drainFD(fd.get(), sink); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 582c78d14..bcb86cbce 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -36,7 +36,7 @@ using namespace nix::daemon; #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { - // We ignore most parameters, we just have them for conformance with the linux syscall + // We ignore most parameters, we just have them for conformance with the linux syscall std::vector buf(8192); auto read_count = read(fd_in, buf.data(), buf.size()); if (read_count == -1) @@ -57,7 +57,7 @@ static void sigChldHandler(int sigNo) { // Ensure we don't modify errno of whatever we've interrupted auto saved_errno = errno; - // Reap all dead children. + // Reap all dead children. while (waitpid(-1, 0, WNOHANG) > 0) ; errno = saved_errno; } @@ -106,7 +106,7 @@ struct PeerInfo }; -// Get the identity of the caller, if possible. +// Get the identity of the caller, if possible. static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -154,12 +154,12 @@ static void daemonLoop(char * * argv) if (chdir("/") == -1) throw SysError("cannot change current directory"); - // Get rid of children automatically; don't let them become zombies. + // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); AutoCloseFD fdSocket; - // Handle socket-based activation by systemd. + // Handle socket-based activation by systemd. auto listenFds = getEnv("LISTEN_FDS"); if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") @@ -168,17 +168,17 @@ static void daemonLoop(char * * argv) closeOnExec(fdSocket.get()); } - // Otherwise, create and bind to a Unix domain socket. + // Otherwise, create and bind to a Unix domain socket. else { createDirs(dirOf(settings.nixDaemonSocketFile)); fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - // Loop accepting connections. + // Loop accepting connections. while (1) { try { - // Accept a connection. + // Accept a connection. struct sockaddr_un remoteAddr; socklen_t remoteAddrLen = sizeof(remoteAddr); @@ -214,7 +214,7 @@ static void daemonLoop(char * * argv) % (peer.pidKnown ? std::to_string(peer.pid) : "") % (peer.uidKnown ? user : "")); - // Fork a child to handle the connection. + // Fork a child to handle the connection. ProcessOptions options; options.errorPrefix = "unexpected Nix daemon error: "; options.dieWithParent = false; @@ -223,20 +223,20 @@ static void daemonLoop(char * * argv) startProcess([&]() { fdSocket = -1; - // Background the daemon. + // Background the daemon. if (setsid() == -1) throw SysError("creating a new session"); - // Restore normal handling of SIGCHLD. + // Restore normal handling of SIGCHLD. setSigChldAction(false); - // For debugging, stuff the pid into argv[1]. + // For debugging, stuff the pid into argv[1]. if (peer.pidKnown && argv[1]) { string processName = std::to_string(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } - // Handle the connection. + // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); @@ -263,7 +263,7 @@ static int _main(int argc, char * * argv) parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") - ; // ignored for backwards compatibility + ; // ignored for backwards compatibility else if (*arg == "--help") showManPage("nix-daemon"); else if (*arg == "--version") @@ -278,7 +278,7 @@ static int _main(int argc, char * * argv) if (stdio) { if (getStoreType() == tDaemon) { - // Forward on this connection to the real daemon + // Forward on this connection to the real daemon auto socketPath = settings.nixDaemonSocketFile; auto s = socket(PF_UNIX, SOCK_STREAM, 0); if (s == -1) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 7ab5ba500..2c27d97f5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -123,7 +123,7 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - logError({ + logError({ .name = "Name collision", .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) }); @@ -874,7 +874,7 @@ static void queryJSON(Globals & globals, vector & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - logError({ + logError({ .name = "Invalid meta attribute", .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j) @@ -1128,8 +1128,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) XMLAttrs attrs2; attrs2["name"] = j; Value * v = i.queryMeta(j); - if (!v) - logError({ + if (!v) + logError({ .name = "Invalid meta attribute", .hint = hintfmt( "derivation '%s' has invalid meta attribute '%s'", diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index dea53b52f..6d2139526 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -704,7 +704,7 @@ static void opVerify(Strings opFlags, Strings opArgs) else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - logWarning({ + logWarning({ .name = "Store consistency", .description = "not all errors were fixed" }); @@ -729,7 +729,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - logError({ + logError({ .name = "Hash mismatch", .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 246b4b480..001401ac2 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -99,12 +99,12 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - logError({ + logError({ .name = "Hash error - path modified", .hint = hintfmt( "path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), + info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true)) }); } @@ -154,7 +154,7 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - logError({ + logError({ .name = "Untrusted path", .hint = hintfmt("path '%s' is untrusted", store->printStorePath(info->path)) diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 82feacb3d..434ad80a6 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,7 +39,7 @@ std::set runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - logError({ + logError({ .name = "Regular MACH file", .hint = hintfmt("file '%s' is not a regular file", filename) }); @@ -47,7 +47,7 @@ std::set runResolver(const Path & filename) } if (st.st_size < sizeof(mach_header_64)) { - logError({ + logError({ .name = "File too short", .hint = hintfmt("file '%s' is too short for a MACH binary", filename) }); @@ -72,7 +72,7 @@ std::set runResolver(const Path & filename) } } if (mach64_offset == 0) { - logError({ + logError({ .name = "No mach64 blobs", .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) }); @@ -81,7 +81,7 @@ std::set runResolver(const Path & filename) } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - logError({ + logError({ .name = "Magic number", .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) }); From 5ed5d7acbd00298df7b4e3a638d731143084da2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 16:03:29 +0200 Subject: [PATCH 336/350] Improve "waiting for locks" messages These are now shown in the progress bar. Closes #3577. --- src/libmain/progress-bar.cc | 2 +- src/libstore/build.cc | 22 +++++++++++++++++++--- src/libutil/logging.hh | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 8d0421c2f..6dfc3a4d3 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -156,7 +156,7 @@ public: { auto state(state_.lock()); - if (lvl <= verbosity && !s.empty()) + if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); state->activities.emplace_back(ActInfo()); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index de393e837..2eb78591b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -869,6 +869,9 @@ private: std::unique_ptr act; + /* Activity that denotes waiting for a lock. */ + std::unique_ptr actLock; + std::map builderActivities; /* The remote machine on which we're building. */ @@ -1439,10 +1442,15 @@ void DerivationGoal::tryToBuild() lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } + actLock.reset(); + /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1481,6 +1489,7 @@ void DerivationGoal::tryToBuild() case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + actLock.reset(); result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); @@ -1488,6 +1497,9 @@ void DerivationGoal::tryToBuild() case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1497,6 +1509,8 @@ void DerivationGoal::tryToBuild() } } + actLock.reset(); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1524,7 +1538,9 @@ void DerivationGoal::tryLocalBuild() { uid. */ buildUser->kill(); } else { - debug("waiting for build users"); + if (!actLock) + actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -1535,6 +1551,8 @@ void DerivationGoal::tryLocalBuild() { #endif } + actLock.reset(); + try { /* Okay, we have to build. */ @@ -4863,8 +4881,6 @@ void Worker::waitForInput() up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printInfo("waiting for locks, build slots or build users..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; timeout = std::max(1L, (long) std::chrono::duration_cast( diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index b99b246c3..b1583eced 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -18,6 +18,7 @@ typedef enum { actSubstitute = 108, actQueryPathInfo = 109, actPostBuildHook = 110, + actBuildWaiting = 111, } ActivityType; typedef enum { From ccfa6b3eee9f5e613152a4b373f67ff05696adf8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 16:12:27 +0200 Subject: [PATCH 337/350] Give better error message about <...> in pure eval mode --- src/libexpr/parser.y | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index a1188dec0..a639be64e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -689,7 +689,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos if (pathExists(res)) return canonPath(res); } throw ThrownError({ - .hint = hintfmt("file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), + .hint = hintfmt(evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path), .nixCode = NixCode { .errPos = pos } }); } From 31707735b699db6dfd78e8bb8e655c3d5e6e0ba5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 16:47:21 +0200 Subject: [PATCH 338/350] Remove unnecessary amDone() overrides --- src/libstore/build.cc | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2eb78591b..2a34cfbb7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -187,7 +187,7 @@ public: protected: - virtual void amDone(ExitCode result); + void amDone(ExitCode result); }; @@ -1009,11 +1009,6 @@ private: void repairClosure(); - void amDone(ExitCode result) override - { - Goal::amDone(result); - } - void started(); void done(BuildResult::Status status, const string & msg = ""); @@ -4335,11 +4330,6 @@ public: void handleEOF(int fd) override; StorePath getStorePath() { return storePath.clone(); } - - void amDone(ExitCode result) override - { - Goal::amDone(result); - } }; From 4e995bc8a66e5486be83c935bd939726ac39cab3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 18:01:05 +0200 Subject: [PATCH 339/350] Always hide the progress bar on exit --- src/libmain/progress-bar.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 6dfc3a4d3..95a9187de 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -111,10 +111,7 @@ public: auto state(state_.lock()); if (!state->active) return; state->active = false; - std::string status = getStatus(*state); writeToStderr("\r\e[K"); - if (status != "") - writeToStderr("[" + status + "]\n"); updateCV.notify_one(); quitCV.notify_one(); } From f20bb983cab168bd9fe4f4ad560a64aeb4b19f07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 18:16:03 +0200 Subject: [PATCH 340/350] Cleanup --- src/libutil/error.cc | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index cb5fe70bf..da11d1ffd 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -105,7 +105,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) { - int errwidth = 80; + size_t errwidth = 80; string prefix = ""; string levelString; @@ -158,12 +158,10 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) } } - int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); - int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; + auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; - string dashes; - for (int i = 0; i < dashwidth; ++i) - dashes.append("-"); + std::string dashes(dashwidth, '-'); // divider. if (einfo.name != "") From 8b099812eab7ccad9ebc6906450b637044de9369 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 18:18:02 +0200 Subject: [PATCH 341/350] Respect terminal width printing error messages --- src/libutil/error.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index da11d1ffd..0fad9ae42 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -105,7 +105,7 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) { - size_t errwidth = 80; + auto errwidth = std::max(getWindowSize().second, 20); string prefix = ""; string levelString; From 24a3208247c248956c5815554804da1bf9763ece Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 18:20:05 +0200 Subject: [PATCH 342/350] Include only the base name of the program in error messages --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0f2c189a6..dc6d5e413 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -290,7 +290,7 @@ int handleExceptions(const string & programName, std::function fun) { ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this - ErrorInfo::programName = programName; + ErrorInfo::programName = baseNameOf(programName); string error = ANSI_RED "error:" ANSI_NORMAL " "; try { From a588b6b19d93fe80deb8518562b7a483fd4c50bc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Jun 2020 19:25:35 +0200 Subject: [PATCH 343/350] Print only one error message if a build fails E.g. instead of error: --- BuildError ----------------------------------------------- nix builder for '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed with exit code 1 error: --- Error ---------------------------------------------------- nix build of '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed we now get error: --- Error ---------------------------------------------------- nix builder for '/nix/store/03nk0a3n8h2948k4lqfgnnmym7knkcma-foo.drv' failed with exit code 1 --- src/libstore/build.cc | 138 ++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 71 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 2a34cfbb7..8ad2ca4f3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -104,13 +104,10 @@ typedef std::map WeakGoalMap; -class Goal : public std::enable_shared_from_this +struct Goal : public std::enable_shared_from_this { -public: typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; -protected: - /* Backlink to the worker. */ Worker & worker; @@ -138,6 +135,9 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + /* Exception containing an error message, if any. */ + std::optional ex; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -149,7 +149,6 @@ protected: trace("goal destroyed"); } -public: virtual void work() = 0; void addWaitee(GoalPtr waitee); @@ -173,21 +172,14 @@ public: return name; } - ExitCode getExitCode() - { - return exitCode; - } - /* Callback in case of a timeout. It should wake up its waiters, get rid of any running child processes that are being monitored by the worker (important!), etc. */ - virtual void timedOut() = 0; + virtual void timedOut(Error && ex) = 0; virtual string key() = 0; -protected: - - void amDone(ExitCode result); + void amDone(ExitCode result, std::optional ex = {}); }; @@ -392,8 +384,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - trace(format("waitee '%1%' done; %2% left") % - waitee->name % waitees.size()); + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; @@ -418,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } -void Goal::amDone(ExitCode result) +void Goal::amDone(ExitCode result, std::optional ex) { trace("done"); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + for (auto & i : waiters) { GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); @@ -910,7 +909,7 @@ public: /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); - void timedOut() override; + void timedOut(Error && ex) override; string key() override { @@ -1011,7 +1010,9 @@ private: void started(); - void done(BuildResult::Status status, const string & msg = ""); + void done( + BuildResult::Status status, + std::optional ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); }; @@ -1105,10 +1106,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::timedOut() +void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut); + done(BuildResult::TimedOut, ex); } @@ -1156,11 +1157,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - logError({ - .name = "missing derivation during build", - .hint = hintfmt("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)) - }); - done(BuildResult::MiscFailure); + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -1349,13 +1346,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - logError({ - .name = "Dependencies could not be built", - .hint = hintfmt( - "cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed) - }); - done(BuildResult::DependencyFailed); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); return; } @@ -1554,11 +1547,10 @@ void DerivationGoal::tryLocalBuild() { startBuilder(); } catch (BuildError & e) { - logError(e.info()); outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); + done(BuildResult::InputRejected, e); return; } @@ -1670,7 +1662,7 @@ void DerivationGoal::buildDone() } auto msg = fmt("builder for '%s' %s", - worker.store.printStorePath(drvPath), + yellowtxt(worker.store.printStorePath(drvPath)), statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { @@ -1771,8 +1763,6 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - logError(e.info()); - outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1791,7 +1781,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e.msg()); + done(st, e); return; } @@ -3881,7 +3871,6 @@ void DerivationGoal::registerOutputs() .hint = hint }); - curRound = nrRounds; // we know enough, bail out early } } @@ -4145,14 +4134,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - logError({ - .name = "Max log size exceeded", - .hint = hintfmt( - "%1% killed after writing more than %2% bytes of log output", - getName(), settings.maxLogSize) - }); killChild(); - done(BuildResult::LogLimitExceeded); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); return; } @@ -4233,11 +4219,12 @@ void DerivationGoal::addHashRewrite(const StorePath & path) } -void DerivationGoal::done(BuildResult::Status status, const string & msg) +void DerivationGoal::done(BuildResult::Status status, std::optional ex) { result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) @@ -4306,7 +4293,7 @@ public: SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); - void timedOut() override { abort(); }; + void timedOut(Error && ex) override { abort(); }; string key() override { @@ -4693,7 +4680,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -4935,32 +4922,24 @@ void Worker::waitForInput() } } - if (goal->getExitCode() == Goal::ecBusy && + if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - logError({ - .name = "Silent build timeout", - .hint = hintfmt( + goal->timedOut(Error( "%1% timed out after %2% seconds of silence", - goal->getName(), settings.maxSilentTime) - }); - goal->timedOut(); + goal->getName(), settings.maxSilentTime)); } - else if (goal->getExitCode() == Goal::ecBusy && + else if (goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - logError({ - .name = "Build timeout", - .hint = hintfmt( + goal->timedOut(Error( "%1% timed out after %2% seconds", - goal->getName(), settings.buildTimeout) - }); - goal->timedOut(); + goal->getName(), settings.buildTimeout)); } } @@ -5066,16 +5045,28 @@ void LocalStore::buildPaths(const std::vector & drvPaths, worker.run(goals); StorePathSet failed; + std::optional ex; for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast(i.get())->getStorePath()); } } - if (!failed.empty()) + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } } BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -5111,8 +5102,13 @@ void LocalStore::ensurePath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } } @@ -5124,7 +5120,7 @@ void LocalStore::repairPath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) { + if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); From 7e7e3b71f33bfe4899d2cf735bf5e7a25bab11fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jun 2020 23:35:07 +0000 Subject: [PATCH 344/350] Add mising #include for strerror --- src/libutil/error.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 2ba3a3b33..1e6102ce1 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -4,6 +4,7 @@ #include "ref.hh" #include "types.hh" +#include #include #include #include From cd8214c39870cc37d10c439978c8cffc69db41e0 Mon Sep 17 00:00:00 2001 From: Tobias Pflug Date: Tue, 16 Jun 2020 09:44:19 +0200 Subject: [PATCH 345/350] Fix logging unit tests --- src/libutil/tests/logging.cc | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index fbdc91253..4cb54995b 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -22,7 +22,7 @@ namespace nix { logger->logEI(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\nan error for testing purposes\n"); + ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\nan error for testing purposes\n"); } } @@ -42,7 +42,7 @@ namespace nix { logger->logEI(ei); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError ------------------------------------ error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); } } @@ -60,7 +60,7 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError ------------------------------------- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); } } @@ -74,7 +74,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name ------------------------------------- error-unit-test\x1B[0m\nInfo description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name --- error-unit-test\x1B[0m\nInfo description\n"); } TEST(logEI, loggingErrorOnTalkativeLevel) { @@ -88,7 +88,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name -------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name --- error-unit-test\x1B[0m\nTalkative description\n"); } TEST(logEI, loggingErrorOnChattyLevel) { @@ -102,7 +102,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name ----------------------------------- error-unit-test\x1B[0m\nTalkative description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name --- error-unit-test\x1B[0m\nTalkative description\n"); } TEST(logEI, loggingErrorOnDebugLevel) { @@ -116,7 +116,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name ----------------------------------- error-unit-test\x1B[0m\nDebug description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name --- error-unit-test\x1B[0m\nDebug description\n"); } TEST(logEI, loggingErrorOnVomitLevel) { @@ -130,7 +130,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name ----------------------------------- error-unit-test\x1B[0m\nVomit description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name --- error-unit-test\x1B[0m\nVomit description\n"); } /* ---------------------------------------------------------------------------- @@ -147,7 +147,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name ----------------------------------------- error-unit-test\x1B[0m\nerror description\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n"); } TEST(logError, logErrorWithPreviousAndNextLinesOfCode) { @@ -171,7 +171,7 @@ namespace nix { auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } TEST(logError, logErrorWithoutLinesOfCode) { @@ -190,7 +190,7 @@ namespace nix { }}); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } TEST(logError, logErrorWithOnlyHintAndName) { @@ -206,7 +206,7 @@ namespace nix { }}); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name ----------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n"); } @@ -224,7 +224,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --------------------------------------- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); } TEST(logWarning, logWarningWithFileLineNumAndCode) { @@ -249,7 +249,7 @@ namespace nix { auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name ------------------------------- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } } From 72e17290d457a8e911f126d21b74c315b034189d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Jun 2020 11:51:34 +0200 Subject: [PATCH 346/350] Fix FTP support Fixes #3618. --- src/libstore/filetransfer.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 601cb858a..c954ace7f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -72,6 +72,17 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + /* Get the HTTP status code, or 0 for other protocols. */ + long getHTTPStatus() + { + long httpStatus = 0; + long protocol = 0; + curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol); + if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS) + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + return httpStatus; + } + TransferItem(curlFileTransfer & fileTransfer, const FileTransferRequest & request, Callback && callback) @@ -83,8 +94,7 @@ struct curlFileTransfer : public FileTransfer , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); /* Only write data to the sink if this is a successful response. */ @@ -316,8 +326,7 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); char * effectiveUriCStr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); @@ -344,7 +353,7 @@ struct curlFileTransfer : public FileTransfer failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); From 759947bf72c134592f0ce23d385e48095bd0a301 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Jun 2020 14:16:39 +0200 Subject: [PATCH 347/350] StorePath: Rewrite in C++ On nix-env -qa -f '', this reduces maximum RSS by 20970 KiB and runtime by 0.8%. This is mostly because we're not parsing the hash part as a hash anymore (just validating that it consists of base-32 characters). Also, replace storePathToHash() by StorePath::hashPart(). --- Makefile | 1 - src/libexpr/local.mk | 2 +- src/libfetchers/local.mk | 2 +- src/libstore/binary-cache-store.cc | 6 +- src/libstore/local-store.cc | 10 ++-- src/libstore/local.mk | 2 +- src/libstore/nar-info-disk-cache.cc | 6 +- src/libstore/nar-info.cc | 4 +- src/libstore/path.cc | 70 +++++++++++------------ src/libstore/path.hh | 72 +++++++++++++++--------- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/store-api.cc | 14 +---- src/libstore/store-api.hh | 4 -- src/libutil/local.mk | 2 - src/libutil/rust-ffi.cc | 2 + src/libutil/rust-ffi.hh | 2 + src/nix/local.mk | 2 +- src/nix/make-content-addressable.cc | 4 +- src/nix/verify.cc | 2 +- src/nix/why-depends.cc | 10 ++-- src/resolve-system-dependencies/local.mk | 2 +- 21 files changed, 110 insertions(+), 111 deletions(-) diff --git a/Makefile b/Makefile index 0ba011e2a..332e6e971 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ - nix-rust/local.mk \ src/libutil/local.mk \ src/libutil/tests/local.mk \ src/libstore/local.mk \ diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 917e8a1c7..9ed39e745 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -libexpr_LIBS = libutil libstore libfetchers libnixrust +libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index d7143d8a6..cfd705e22 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,4 +8,4 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LIBS = libutil libstore libnixrust +libfetchers_LIBS = libutil libstore diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 649331495..1037b2e28 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -93,7 +93,7 @@ std::shared_ptr BinaryCacheStore::getFile(const std::string & path) std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - return storePathToHash(printStorePath(storePath)) + ".narinfo"; + return std::string(storePath.hashPart()) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref narInfo) @@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(printStorePath(narInfo->path)); + std::string hashPart(narInfo->path.hashPart()); { auto state_(state.lock()); @@ -164,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } } - upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f5c5bd9b7..e3b718e88 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -619,7 +619,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), + state_->pathInfoCache.upsert(std::string(info.path.hashPart()), PathInfoCacheValue{ .value = std::make_shared(info) }); } @@ -791,7 +791,7 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { - if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); + if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; @@ -942,7 +942,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); + state_->pathInfoCache.erase(std::string(path.hashPart())); } } @@ -994,7 +994,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique(htSHA256); else - hashSink = std::make_unique(htSHA256, storePathToHash(printStorePath(info.path))); + hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); @@ -1255,7 +1255,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->ca == "" || !info->references.count(info->path)) hashSink = std::make_unique(info->narHash.type); else - hashSink = std::make_unique(info->narHash.type, storePathToHash(printStorePath(info->path))); + hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 91acef368..aec4ed493 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index e8cf1d177..552970248 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,7 +189,7 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref(StorePath::fromBaseName(hashPart + "-" + namePart)); + auto narInfo = make_ref(StorePath(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -198,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath::fromBaseName(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); + narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = queryNAR.getStr(11); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 232284723..d7fc30e91 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath::fromBaseName(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") - deriver = StorePath::fromBaseName(value); + deriver = StorePath(value); } else if (name == "System") system = value; diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 9a28aa96a..bb2089ea4 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,38 +2,38 @@ namespace nix { -extern "C" { - rust::Result ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); - rust::Result ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); - rust::Result ffi_StorePath_fromBaseName(rust::StringSlice baseName); - rust::String ffi_StorePath_to_string(const StorePath & _this); - StorePath ffi_StorePath_clone(const StorePath & _this); - rust::StringSlice ffi_StorePath_name(const StorePath & _this); +MakeError(BadStorePath, Error); + +static void checkName(std::string_view path, std::string_view name) +{ + if (name.empty()) + throw BadStorePath("store path '%s' has an empty name", path); + if (name.size() > 211) + throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + for (auto c : name) + if (!((c >= '0' && c <= '9') + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) + throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); } -StorePath StorePath::make(std::string_view path, std::string_view storeDir) +StorePath::StorePath(std::string_view _baseName) + : baseName(_baseName) { - return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); + if (baseName.size() < HashLen + 1) + throw BadStorePath("'%s' is too short to be a valid store path", baseName); + for (auto c : hashPart()) + if (c == 'e' || c == 'o' || c == 'u' || c == 't' + || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) + throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); + checkName(baseName, name()); } -StorePath StorePath::make(unsigned char hash[20], std::string_view name) +StorePath::StorePath(const Hash & hash, std::string_view _name) + : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) { - return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); -} - -StorePath StorePath::fromBaseName(std::string_view baseName) -{ - return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); -} - -rust::String StorePath::to_string() const -{ - return ffi_StorePath_to_string(*this); -} - -StorePath StorePath::clone() const -{ - return ffi_StorePath_clone(*this); + checkName(baseName, name()); } bool StorePath::isDerivation() const @@ -41,18 +41,14 @@ bool StorePath::isDerivation() const return hasSuffix(name(), drvExtension); } -std::string_view StorePath::name() const -{ - return ffi_StorePath_name(*this); -} - -StorePath StorePath::dummy( - StorePath::make( - (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); +StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath Store::parseStorePath(std::string_view path) const { - return StorePath::make(path, storeDir); + auto p = canonPath(std::string(path)); + if (dirOf(p) != storeDir) + throw BadStorePath("path '%s' is not in the Nix store", p); + return StorePath(baseNameOf(p)); } std::optional Store::maybeParseStorePath(std::string_view path) const @@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const std::string Store::printStorePath(const StorePath & path) const { - auto s = storeDir + "/"; - s += (std::string_view) path.to_string(); - return s; + return (storeDir + "/").append(path.to_string()); } PathSet Store::printStorePathSet(const StorePathSet & paths) const diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5122e7422..85c3d8e53 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,59 +1,78 @@ #pragma once -#include "rust-ffi.hh" +#include "types.hh" namespace nix { -/* See path.rs. */ -struct StorePath; - class Store; +struct Hash; -extern "C" { - void ffi_StorePath_drop(void *); - bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); - bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); - unsigned char * ffi_StorePath_hash_data(const StorePath & p); -} - -struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +class StorePath { + std::string baseName; + + StorePath(const StorePath & path) + : baseName(path.baseName) + { } + +public: + + /* Size of the hash part of store paths, in base-32 characters. */ + constexpr static size_t HashLen = 32; // i.e. 160 bits + StorePath() = delete; - static StorePath make(std::string_view path, std::string_view storeDir); + StorePath(std::string_view baseName); - static StorePath make(unsigned char hash[20], std::string_view name); + StorePath(const Hash & hash, std::string_view name); - static StorePath fromBaseName(std::string_view baseName); + StorePath(StorePath && path) + : baseName(std::move(path.baseName)) + { } - rust::String to_string() const; + StorePath & operator = (StorePath && path) + { + baseName = std::move(path.baseName); + return *this; + } + + std::string_view to_string() const + { + return baseName; + } bool operator < (const StorePath & other) const { - return ffi_StorePath_less_than(*this, other); + return baseName < other.baseName; } bool operator == (const StorePath & other) const { - return ffi_StorePath_eq(*this, other); + return baseName == other.baseName; } bool operator != (const StorePath & other) const { - return !(*this == other); + return baseName != other.baseName; } - StorePath clone() const; + StorePath clone() const + { + return StorePath(*this); + } /* Check whether a file name ends with the extension for derivations. */ bool isDerivation() const; - std::string_view name() const; - - unsigned char * hashData() const + std::string_view name() const { - return ffi_StorePath_hash_data(*this); + return std::string_view(baseName).substr(HashLen + 1); + } + + std::string_view hashPart() const + { + return std::string_view(baseName).substr(0, HashLen); } static StorePath dummy; @@ -67,9 +86,6 @@ StorePathSet storePathsToSet(const StorePaths & paths); StorePathSet singleton(const StorePath & path); -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; @@ -107,7 +123,7 @@ namespace std { template<> struct hash { std::size_t operator()(const nix::StorePath & path) const noexcept { - return * (std::size_t *) path.hashData(); + return * (std::size_t *) path.to_string().data(); } }; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 9277a8e6b..bd698d781 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); + return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); } void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e23a9ca50..42e26c427 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -59,14 +59,6 @@ StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view p } -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); -} - - /* Store paths have the following form: /- @@ -144,7 +136,7 @@ StorePath Store::makeStorePath(const string & type, /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); - return StorePath::make(h.hash, name); + return StorePath(h, name); } @@ -243,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow() bool Store::isValidPath(const StorePath & storePath) { - auto hashPart = storePathToHash(printStorePath(storePath)); + std::string hashPart(storePath.hashPart()); { auto state_(state.lock()); @@ -311,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath, std::string hashPart; try { - hashPart = storePathToHash(printStorePath(storePath)); + hashPart = storePath.hashPart(); { auto res = state.lock()->pathInfoCache.get(hashPart); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5ef506326..251fc4638 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -731,10 +731,6 @@ public: }; -/* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); - - /* Copy a path from one store to another. */ void copyStorePath(ref srcStore, ref dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 16c1fa03f..ae7eb67ad 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,5 +7,3 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context - -libutil_LIBS = libnixrust diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index 6f36b3192..67924568f 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,3 +1,4 @@ +#if 0 #include "logging.hh" #include "rust-ffi.hh" @@ -20,3 +21,4 @@ std::ostream & operator << (std::ostream & str, const String & s) } } +#endif diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index 228e2eead..cfbaf9dec 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -1,4 +1,5 @@ #pragma once +#if 0 #include "serialise.hh" @@ -185,3 +186,4 @@ struct Result }; } +#endif diff --git a/src/nix/local.mk b/src/nix/local.mk index 43b7754e3..b057b7cc6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -17,7 +17,7 @@ nix_SOURCES := \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust +nix_LIBS = libexpr libmain libfetchers libstore libutil nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 719ea4fd1..1211dad7b 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -48,7 +48,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON for (auto & path : paths) { auto pathS = store->printStorePath(path); auto oldInfo = store->queryPathInfo(path); - auto oldHashPart = storePathToHash(pathS); + std::string oldHashPart(path.hashPart()); StringSink sink; store->narFromPath(path, sink); @@ -88,7 +88,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); rsink2((unsigned char *) sink.s->data(), sink.s->size()); rsink2.flush(); }); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 001401ac2..ab83637dc 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -90,7 +90,7 @@ struct CmdVerify : StorePathsCommand if (info->ca == "") hashSink = std::make_unique(info->narHash.type); else - hashSink = std::make_unique(info->narHash.type, storePathToHash(store->printStorePath(info->path))); + hashSink = std::make_unique(info->narHash.type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 6057beedb..a4ee2d971 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -76,7 +76,7 @@ struct CmdWhyDepends : SourceExprCommand auto packagePath = toStorePath(store, Build, package); auto dependency = parseInstallable(*this, store, _dependency, false); auto dependencyPath = toStorePath(store, NoBuild, dependency); - auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath)); + auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); @@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand auto & node2 = graph.at(ref); if (node2.dist == inf) continue; refs.emplace(node2.dist, &node2); - hashes.insert(storePathToHash(store->printStorePath(node2.path))); + hashes.insert(std::string(node2.path.hashPart())); } /* For each reference, find the files and symlinks that @@ -211,7 +211,7 @@ struct CmdWhyDepends : SourceExprCommand p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), - pos - pos2, storePathHashLen, + pos - pos2, StorePath::HashLen, getColour(hash)))); } } @@ -224,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand auto pos = target.find(hash); if (pos != std::string::npos) hits[hash].emplace_back(fmt("%s -> %s\n", p2, - hilite(target, pos, storePathHashLen, getColour(hash)))); + hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } }; @@ -235,7 +235,7 @@ struct CmdWhyDepends : SourceExprCommand RunPager pager; for (auto & ref : refs) { - auto hash = storePathToHash(store->printStorePath(ref.second->path)); + std::string hash(ref.second->path.hashPart()); bool last = all ? ref == *refs.rbegin() : true; diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index f0e82e023..054ae01cb 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -8,6 +8,6 @@ resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust +resolve-system-dependencies_LIBS := libstore libmain libutil resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc From cc83a86276726699e5e856d26b5e24ce24f5cf34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Jun 2020 14:33:03 +0200 Subject: [PATCH 348/350] release.nix: Remove vendoredCrates --- release-common.nix | 1 - release.nix | 63 ---------------------------------------------- 2 files changed, 64 deletions(-) diff --git a/release-common.nix b/release-common.nix index 7e7de005d..4316c3c23 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,7 +50,6 @@ rec { libarchive boost nlohmann_json - rustc cargo # Tests git diff --git a/release.nix b/release.nix index 2a320e1c3..fbf9e4721 100644 --- a/release.nix +++ b/release.nix @@ -12,64 +12,8 @@ let builtins.readFile ./.version + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"); - # Create a "vendor" directory that contains the crates listed in - # Cargo.lock. This allows Nix to be built without network access. - vendoredCrates' = - let - lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); - - files = map (pkg: import { - url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; - sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; - }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); - - in pkgs.runCommand "cargo-vendor-dir" {} - '' - mkdir -p $out/vendor - - cat > $out/vendor/config < "$dir/.cargo-checksum.json" - - # Clean up some cruft from the winapi crates. FIXME: find - # a way to remove winapi* from our dependencies. - if [[ $dir =~ /winapi ]]; then - find $dir -name "*.a" -print0 | xargs -0 rm -f -- - fi - - mv "$dir" $out/vendor/ - - rm -rf $out/vendor/tmp - '') files)} - ''; - jobs = rec { - vendoredCrates = - with pkgs; - runCommand "vendored-crates" {} - '' - mkdir -p $out/nix-support - name=nix-vendored-crates-${version} - fn=$out/$name.tar.xz - tar cvfJ $fn -C ${vendoredCrates'} vendor \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --transform "s,vendor,$name," - echo "file crates-tarball $fn" >> $out/nix-support/hydra-build-products - ''; - build = pkgs.lib.genAttrs systems (system: let pkgs = import nixpkgs { inherit system; }; in @@ -101,8 +45,6 @@ let patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} - ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor - (cd perl; autoreconf --install --force --verbose) ''; @@ -247,11 +189,6 @@ let src = nix; - preConfigure = - '' - ln -sfn ${vendoredCrates'}/vendor/ nix-rust/vendor - ''; - enableParallelBuilding = true; buildInputs = buildDeps ++ propagatedDeps; From fbf90bd693f09e6ce6e0725563c7996cba4da896 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jun 2020 14:19:49 +0000 Subject: [PATCH 349/350] Add another missing #include --- src/libutil/config.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 5c7a70a2e..66073546e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,3 +1,4 @@ +#include #include #include From 29542865cee37ab22efe1bd142900b69f6c59f0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Jun 2020 22:20:18 +0200 Subject: [PATCH 350/350] Remove StorePath::clone() and related functions --- src/libexpr/primops.cc | 12 +-- src/libstore/binary-cache-store.cc | 6 +- src/libstore/build.cc | 110 ++++++++++++++-------------- src/libstore/daemon.cc | 8 +- src/libstore/derivations.cc | 42 ++++------- src/libstore/derivations.hh | 8 -- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 24 +++--- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-fs-store.cc | 4 +- src/libstore/local-store.cc | 24 +++--- src/libstore/misc.cc | 36 ++++----- src/libstore/nar-info.cc | 2 +- src/libstore/parsed-derivations.cc | 4 +- src/libstore/parsed-derivations.hh | 2 +- src/libstore/path.cc | 23 ------ src/libstore/path.hh | 36 --------- src/libstore/remote-store.cc | 10 +-- src/libstore/store-api.cc | 21 +----- src/libstore/store-api.hh | 3 +- src/nix-build/nix-build.cc | 8 +- src/nix-env/nix-env.cc | 12 ++- src/nix-env/user-env.cc | 2 +- src/nix-store/dotgraph.cc | 4 +- src/nix-store/graphml.cc | 4 +- src/nix-store/nix-store.cc | 36 ++++----- src/nix/command.cc | 10 +-- src/nix/copy.cc | 2 +- src/nix/develop.cc | 6 +- src/nix/installables.cc | 22 +++--- src/nix/make-content-addressable.cc | 6 +- src/nix/path-info.cc | 2 +- src/nix/run.cc | 6 +- src/nix/why-depends.cc | 6 +- 34 files changed, 201 insertions(+), 304 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 62e5163c9..907f15246 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -55,7 +55,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!decoded.second.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}}); + drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}}); /* Add the output of this derivation to the allowed paths. */ @@ -723,9 +723,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StorePathSet refs; state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) { - drv.inputSrcs.insert(j.clone()); + drv.inputSrcs.insert(j); if (j.isDerivation()) - drv.inputDrvs[j.clone()] = state.store->readDerivation(j).outputNames(); + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); } } @@ -792,7 +792,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput(StorePath::dummy.clone(), "", "")); + DerivationOutput { StorePath::dummy, "", "" }); } Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); @@ -801,7 +801,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeOutputPath(i, h, drvName); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, - DerivationOutput(std::move(outPath), "", "")); + DerivationOutput { std::move(outPath), "", "" }); } } @@ -814,7 +814,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't read them later. */ - drvHashes.insert_or_assign(drvPath.clone(), + drvHashes.insert_or_assign(drvPath, hashDerivationModulo(*state.store, Derivation(drv), false)); state.mkAttrs(v, 1 + drv.outputs.size()); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 1037b2e28..f8eff508c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -360,7 +360,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = cloneStorePathSet(references); + info.references = references; if (repair || !isValidPath(info.path)) { StringSink sink; @@ -395,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe std::shared_ptr BinaryCacheStore::getBuildLog(const StorePath & path) { - auto drvPath = path.clone(); + auto drvPath = path; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo if (!info->deriver) return nullptr; - drvPath = info->deriver->clone(); + drvPath = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8ad2ca4f3..53a0958aa 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -295,7 +295,7 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr makeBasicDerivationGoal(StorePath && drvPath, + std::shared_ptr makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); @@ -347,7 +347,7 @@ public: contents. */ bool pathContentsGood(const StorePath & path); - void markContentsGood(StorePath && path); + void markContentsGood(const StorePath & path); void updateProgress() { @@ -900,9 +900,9 @@ private: friend struct RestrictedStore; public: - DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, + DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); @@ -924,7 +924,7 @@ public: StorePath getDrvPath() { - return drvPath.clone(); + return drvPath; } /* Add wanted outputs to an already existing derivation goal. */ @@ -1021,11 +1021,11 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { @@ -1038,11 +1038,11 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , buildMode(buildMode) { this->drv = std::make_unique(BasicDerivation(drv)); @@ -1193,7 +1193,7 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique(drvPath.clone(), *drv); + parsedDrv = std::make_unique(drvPath, *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1301,7 +1301,7 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); + outputsToDrv.insert_or_assign(j.second.path, i); } /* Check each path (slow!). */ @@ -1454,7 +1454,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = cloneStorePathSet(drv->outputPaths()); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1901,14 +1901,14 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (!inputPaths.count(storePath)) throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(singleton(storePath), paths); + worker.store.computeFSClosure({storePath}, paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - auto paths2 = cloneStorePathSet(paths); + auto paths2 = paths; for (auto & j : paths2) { if (j.isDerivation()) { @@ -2037,7 +2037,7 @@ void DerivationGoal::startBuilder() /* Write closure info to . */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(singleton(storePath)), false, false)); + exportReferences({storePath}), false, false)); } } @@ -2222,7 +2222,7 @@ void DerivationGoal::startBuilder() for (auto & i : missingPaths) if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i.clone()); + redirectedBadOutputs.insert(i); } } @@ -2717,8 +2717,8 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryAllValidPaths() override { StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p.clone()); - for (auto & p : goal.addedPaths) paths.insert(p.clone()); + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); return paths; } @@ -2806,7 +2806,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path.clone()); + newPaths.insert(output.second.path); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2851,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore if (goal.isAllowed(path.path)) allowed.emplace_back(path); else - unknown.insert(path.path.clone()); + unknown.insert(path.path); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2946,7 +2946,7 @@ void DerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; - addedPaths.insert(path.clone()); + addedPaths.insert(path); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ @@ -3568,7 +3568,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path.clone()); + result.insert(drv.outputs.find(i)->second.path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3626,9 +3626,9 @@ void DerivationGoal::registerOutputs() output paths, and any paths that have been built via recursive Nix calls. */ StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); - for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -3827,7 +3827,7 @@ void DerivationGoal::registerOutputs() info.narHash = hash.first; info.narSize = hash.second; info.references = std::move(references); - info.deriver = drvPath.clone(); + info.deriver = drvPath; info.ultimate = true; info.ca = ca; worker.store.signPathInfo(info); @@ -3942,23 +3942,23 @@ void DerivationGoal::checkOutputs(const std::map & outputs) uint64_t closureSize = 0; StorePathSet pathsDone; std::queue pathsLeft; - pathsLeft.push(path.clone()); + pathsLeft.push(path); while (!pathsLeft.empty()) { - auto path = pathsLeft.front().clone(); + auto path = pathsLeft.front(); pathsLeft.pop(); - if (!pathsDone.insert(path.clone()).second) continue; + if (!pathsDone.insert(path).second) continue; auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } } @@ -3985,8 +3985,8 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); auto used = recursive - ? cloneStorePathSet(getClosure(info.path).first) - : cloneStorePathSet(info.references); + ? getClosure(info.path).first + : info.references; if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -3996,10 +3996,10 @@ void DerivationGoal::checkOutputs(const std::map & outputs) for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } else { if (spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } if (!badPaths.empty()) { @@ -4199,7 +4199,7 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path.clone()); + if (good == returnValid) result.insert(i.second.path); } return result; } @@ -4215,7 +4215,7 @@ void DerivationGoal::addHashRewrite(const StorePath & path) deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); + redirectedOutputs.insert_or_assign(path, std::move(p)); } @@ -4290,7 +4290,7 @@ private: GoalState state; public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4316,13 +4316,13 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - StorePath getStorePath() { return storePath.clone(); } + StorePath getStorePath() { return storePath; } }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , storePath(std::move(storePath)) + , storePath(storePath) , repair(repair) { state = &SubstitutionGoal::init; @@ -4556,7 +4556,7 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath.clone()); + worker.markContentsGood(storePath); printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); @@ -4626,10 +4626,10 @@ Worker::~Worker() GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME + GoalPtr goal = derivationGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared(path.clone(), wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared(path, wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path, goal); wakeUp(goal); } else (dynamic_cast(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4637,10 +4637,10 @@ GoalPtr Worker::makeDerivationGoal(const StorePath & path, } -std::shared_ptr Worker::makeBasicDerivationGoal(StorePath && drvPath, +std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared(std::move(drvPath), drv, *this, buildMode); + auto goal = std::make_shared(drvPath, drv, *this, buildMode); wakeUp(goal); return goal; } @@ -4648,10 +4648,10 @@ std::shared_ptr Worker::makeBasicDerivationGoal(StorePath && drv GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME + GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared(path.clone(), *this, repair); - substitutionGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared(path, *this, repair); + substitutionGoals.insert_or_assign(path, goal); wakeUp(goal); } return goal; @@ -4996,7 +4996,7 @@ bool Worker::pathContentsGood(const StorePath & path) Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache.insert_or_assign(path.clone(), res); + pathContentsGoodCache.insert_or_assign(path, res); if (!res) logError({ .name = "Corrupted path", @@ -5006,9 +5006,9 @@ bool Worker::pathContentsGood(const StorePath & path) } -void Worker::markContentsGood(StorePath && path) +void Worker::markContentsGood(const StorePath & path) { - pathContentsGoodCache.insert_or_assign(std::move(path), true); + pathContentsGoodCache.insert_or_assign(path, true); } @@ -5073,7 +5073,7 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); BuildResult result; @@ -5094,7 +5094,7 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {StorePathWithOutputs(path)}); + primeCache(*this, {{path}}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 620722516..e370e278c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -293,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME - paths.insert(path.clone()); + paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); to << (res.count(path) != 0); @@ -327,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref store, StorePathSet paths; if (op == wopQueryReferences) for (auto & i : store->queryPathInfo(path)->references) - paths.insert(i.clone()); + paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) @@ -593,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - StorePathSet paths; - paths.insert(path.clone()); // FIXME - store->querySubstitutablePathInfos(paths, infos); + store->querySubstitutablePathInfos({path}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 915e02eed..b95f7bfdc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & has } -BasicDerivation::BasicDerivation(const BasicDerivation & other) - : platform(other.platform) - , builder(other.builder) - , args(other.args) - , env(other.env) -{ - for (auto & i : other.outputs) - outputs.insert_or_assign(i.first, - DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); - for (auto & i : other.inputSrcs) - inputSrcs.insert(i.clone()); -} - - -Derivation::Derivation(const Derivation & other) - : BasicDerivation(other) -{ - for (auto & i : other.inputDrvs) - inputDrvs.insert_or_assign(i.first.clone(), i.second); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -67,9 +45,9 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref store, const Derivation & drv, std::string_view name, RepairFlag repair) { - auto references = cloneStorePathSet(drv.inputSrcs); + auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) - references.insert(i.first.clone()); + references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ @@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s) expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(id, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } /* Parse the list of input derivations. */ @@ -383,7 +365,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput auto h = drvHashes.find(i.first); if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, + h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); @@ -411,7 +393,7 @@ StorePathSet BasicDerivation::outputPaths() const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path.clone()); + paths.insert(i.second.path); return paths; } @@ -434,7 +416,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto path = store.parseStorePath(readString(in)); auto hashAlgo = readString(in); auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(name, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } drv.inputSrcs = readStorePaths(store, in); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 88aed66bf..d349c6d4d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,11 +17,6 @@ struct DerivationOutput StorePath path; std::string hashAlgo; /* hash used for expected hash computation */ std::string hash; /* expected hash, may be null */ - DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) - : path(std::move(path)) - , hashAlgo(std::move(hashAlgo)) - , hash(std::move(hash)) - { } void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; @@ -43,7 +38,6 @@ struct BasicDerivation StringPairs env; BasicDerivation() { } - explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in @@ -71,8 +65,6 @@ struct Derivation : BasicDerivation std::map * actualInputs = nullptr) const; Derivation() { } - Derivation(Derivation && other) = default; - explicit Derivation(const Derivation & other); }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 54471d4a3..cb9da027d 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -105,7 +105,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces auto source = StringSource { *tee.source.data }; addToStore(info, source, NoRepair, checkSigs, accessor); - res.push_back(info.path.clone()); + res.push_back(info.path); } return res; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 04e3849f7..57fb20845 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -128,7 +128,7 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, gcroots directory. */ if (settings.checkRootReachability) { auto roots = findRoots(false); - if (roots[storePath.clone()].count(gcRoot) == 0) + if (roots[storePath].count(gcRoot) == 0) logWarning({ .name = "GC root", .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -478,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!isValidPath(path)) continue; debug("got additional root '%1%'", pathS); if (censor) - roots[path.clone()].insert(censored); + roots[path].insert(censored); else - roots[path.clone()].insert(links.begin(), links.end()); + roots[path].insert(links.begin(), links.end()); } } @@ -592,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.roots.count(path)) { debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } - visited.insert(path.clone()); + visited.insert(path); if (!isValidPath(path)) return false; @@ -610,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepDerivations && path.isDerivation()) { for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i.clone()); + incoming.insert(i); } /* If keep-outputs is set, then don't delete this path if there @@ -618,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepOutputs) { auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i.clone()); + incoming.insert(i); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } @@ -668,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ for (auto & i : visited) - state.dead.insert(i.clone()); + state.dead.insert(i); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -754,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first.clone()); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -763,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Roots tempRoots; findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) { - state.tempRoots.insert(root.first.clone()); - state.roots.insert(root.first.clone()); + state.tempRoots.insert(root.first); + state.roots.insert(root.first); } /* After this point the set of roots or temporary roots cannot diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index af20d389b..45c70fad6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -256,7 +256,7 @@ struct LegacySSHStore : public Store conn->to.flush(); for (auto & i : readStorePaths(*this, conn->from)) - out.insert(i.clone()); + out.insert(i); } StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 2d564a0d7..dd96d2578 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs"; std::shared_ptr LocalFSStore::getBuildLog(const StorePath & path_) { - auto path = path_.clone(); + auto path = path_; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); if (!info->deriver) return nullptr; - path = info->deriver->clone(); + path = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e3b718e88..0df4374b9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -631,7 +631,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - auto info = std::make_shared(path.clone()); + auto info = std::make_shared(path); callback(retrySQLite>([&]() { auto state(_state.lock()); @@ -721,7 +721,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } @@ -816,7 +816,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining; for (auto & i : paths) - remaining.insert(i.clone()); + remaining.insert(i); StorePathSet res; @@ -830,9 +830,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path.clone()); + res.insert(path); else - remaining2.insert(path.clone()); + remaining2.insert(path); std::swap(remaining, remaining2); } @@ -854,9 +854,9 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ - info->deriver ? info->deriver->clone() : std::optional(), - cloneStorePathSet(info->references), + infos.insert_or_assign(path, SubstitutablePathInfo{ + info->deriver, + info->references, narInfo ? narInfo->fileSize : 0, info->narSize}); } catch (InvalidPath &) { @@ -900,7 +900,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path.clone()); + paths.insert(i.path); } for (auto & i : infos) { @@ -1074,7 +1074,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; info.ca = makeFixedOutputCA(method, h); @@ -1137,10 +1137,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = cloneStorePathSet(references); + info.references = references; info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..e68edb38c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { StorePathSet paths; - paths.insert(startPath.clone()); + paths.insert(startPath); computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } @@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector & targets, auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath.clone()); + state->willBuild.insert(drvPath); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); }; auto checkOutput = [&]( @@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({outPath}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath.clone()); + drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); } } } @@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector & targets, if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } auto drv = make_ref(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(path.path.clone(), *drv); + ParsedDerivation parsedDrv(StorePath(path.path), *drv); PathSet invalid; for (auto & j : drv->outputs) @@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(path.path.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({path.path}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } @@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector & targets, { auto state(state_.lock()); - state->willSubstitute.insert(path.path.clone()); + state->willSubstitute.insert(path.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); } }; @@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) throw BuildError("cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path.clone()).second) return; - parents.insert(path.clone()); + if (!visited.insert(path).second) return; + parents.insert(path); StorePathSet references; try { - references = cloneStorePathSet(queryPathInfo(path)->references); + references = queryPathInfo(path)->references; } catch (InvalidPath &) { } @@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_back(path.clone()); + sorted.push_back(path); parents.erase(path); }; diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d7fc30e91..bb4448c90 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,7 +4,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { throw Error("NAR info file '%1%' is corrupt", whence); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 45c033c66..24f848e46 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -4,8 +4,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) - : drvPath(std::move(drvPath)), drv(drv) +ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index f4df5dd54..7621342d7 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -12,7 +12,7 @@ class ParsedDerivation public: - ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); + ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); ~ParsedDerivation(); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index bb2089ea4..b3d8ce95c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -84,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -StorePathSet cloneStorePathSet(const StorePathSet & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet storePathsToSet(const StorePaths & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet singleton(const StorePath & path) -{ - StorePathSet res; - res.insert(path.clone()); - return res; -} - std::pair parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 85c3d8e53..aaebd3ec3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -11,10 +11,6 @@ class StorePath { std::string baseName; - StorePath(const StorePath & path) - : baseName(path.baseName) - { } - public: /* Size of the hash part of store paths, in base-32 characters. */ @@ -26,16 +22,6 @@ public: StorePath(const Hash & hash, std::string_view name); - StorePath(StorePath && path) - : baseName(std::move(path.baseName)) - { } - - StorePath & operator = (StorePath && path) - { - baseName = std::move(path.baseName); - return *this; - } - std::string_view to_string() const { return baseName; @@ -56,11 +42,6 @@ public: return baseName != other.baseName; } - StorePath clone() const - { - return StorePath(*this); - } - /* Check whether a file name ends with the extension for derivations. */ bool isDerivation() const; @@ -81,11 +62,6 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -StorePathSet cloneStorePathSet(const StorePathSet & paths); -StorePathSet storePathsToSet(const StorePaths & paths); - -StorePathSet singleton(const StorePath & path); - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; @@ -99,18 +75,6 @@ struct StorePathWithOutputs StorePath path; std::set outputs; - StorePathWithOutputs(const StorePath & path, const std::set & outputs = {}) - : path(path.clone()), outputs(outputs) - { } - - StorePathWithOutputs(StorePath && path, std::set && outputs) - : path(std::move(path)), outputs(std::move(outputs)) - { } - - StorePathWithOutputs(const StorePathWithOutputs & other) - : path(other.path.clone()), outputs(other.outputs) - { } - std::string to_string(const Store & store) const; }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 99fee8150..fc5ab5865 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } else { conn->to << wopQueryValidPaths; @@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & i : paths) { conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i.clone()); + if (readInt(conn->from)) res.insert(i); } return res; } else { @@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.clone(), std::move(info)); + infos.insert_or_assign(i, std::move(info)); } } else { @@ -372,7 +372,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared(path.clone()); + info = std::make_shared(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); @@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path, conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); for (auto & i : readStorePaths(*this, conn->from)) - referrers.insert(i.clone()); + referrers.insert(i); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 42e26c427..aae227bae 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,7 +55,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const { auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs)); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; } @@ -545,7 +545,7 @@ void Store::buildPaths(const std::vector & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) unsupported("buildPaths"); - paths2.insert(path.path.clone()); + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) @@ -685,21 +685,6 @@ void copyClosure(ref srcStore, ref dstStore, } -ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) - : path(other.path.clone()) - , deriver(other.deriver ? other.deriver->clone(): std::optional{}) - , narHash(other.narHash) - , references(cloneStorePathSet(other.references)) - , registrationTime(other.registrationTime) - , narSize(other.narSize) - , id(other.id) - , ultimate(other.ultimate) - , sigs(other.sigs) - , ca(other.ca) -{ -} - - std::optional decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) { std::string path; @@ -785,7 +770,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); - auto refs = cloneStorePathSet(references); + auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 251fc4638..6f4dd959c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,8 +189,9 @@ struct ValidPathInfo Strings shortRefs() const; + ValidPathInfo(const StorePath & path) : path(path) { } + ValidPathInfo(StorePath && path) : path(std::move(path)) { } - explicit ValidPathInfo(const ValidPathInfo & other); virtual ~ValidPathInfo() { } }; diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index da6f005a3..a224d635d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -363,7 +363,7 @@ static void _main(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); - pathsToBuild.emplace_back(store->parseStorePath(drv->queryDrvPath())); + pathsToBuild.push_back({store->parseStorePath(drv->queryDrvPath())}); shell = drv->queryOutPath() + "/bin/bash"; @@ -381,9 +381,9 @@ static void _main(int argc, char * * argv) for (const auto & input : drv.inputDrvs) if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) - pathsToBuild.emplace_back(input.first, input.second); + pathsToBuild.push_back({input.first, input.second}); for (const auto & src : drv.inputSrcs) - pathsToBuild.emplace_back(src); + pathsToBuild.push_back({src}); buildPaths(pathsToBuild); @@ -499,7 +499,7 @@ static void _main(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); - pathsToBuild.emplace_back(store->parseStorePath(drvPath), StringSet{outputName}); + pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}}); std::string drvPrefix; auto i = drvPrefixes.find(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 2c27d97f5..8b0692035 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -212,9 +212,7 @@ static bool isPrebuilt(EvalState & state, DrvInfo & elem) { auto path = state.store->parseStorePath(elem.queryOutPath()); if (state.store->isValidPath(path)) return true; - StorePathSet paths; - paths.insert(path.clone()); // FIXME: why doesn't StorePathSet{path.clone()} work? - return state.store->querySubstitutablePaths(paths).count(path); + return state.store->querySubstitutablePaths({path}).count(path); } @@ -425,9 +423,9 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.emplace_back(state.store->parseStorePath(drvPath)); + targets.push_back({state.store->parseStorePath(drvPath)}); else - targets.emplace_back(state.store->parseStorePath(i.queryOutPath())); + targets.push_back({state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -697,13 +695,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) drv.setName(globals.forceName); if (drv.queryDrvPath() != "") { - std::vector paths{globals.state->store->parseStorePath(drv.queryDrvPath())}; + std::vector paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); } else { printMissing(globals.state->store, - {globals.state->store->parseStorePath(drv.queryOutPath())}); + {{globals.state->store->parseStorePath(drv.queryOutPath())}}); if (globals.dryRun) return; globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index f804b77a0..8e7f09e12 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -134,7 +134,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Realise the resulting store expression. */ debug("building user environment"); std::vector topLevelDrvs; - topLevelDrvs.push_back(StorePathWithOutputs{topLevelDrv.clone()}); + topLevelDrvs.push_back({topLevelDrv}); state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 667d917f5..8b699f39b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -54,13 +54,13 @@ void printDotGraph(ref store, StorePathSet && roots) while (!workList.empty()) { auto path = std::move(workList.extract(workList.begin()).value()); - if (!doneSet.insert(path.clone()).second) continue; + if (!doneSet.insert(path).second) continue; cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { - workList.insert(p.clone()); + workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); } } diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 347708851..8ca5c9c8d 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -65,7 +65,7 @@ void printGraphML(ref store, StorePathSet && roots) while (!workList.empty()) { auto path = std::move(workList.extract(workList.begin()).value()); - ret = doneSet.insert(path.clone()); + ret = doneSet.insert(path); if (ret.second == false) continue; auto info = store->queryPathInfo(path); @@ -73,7 +73,7 @@ void printGraphML(ref store, StorePathSet && roots) for (auto & p : info->references) { if (p != path) { - workList.insert(p.clone()); + workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6d2139526..5c5afd5ec 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -49,11 +49,11 @@ ref ensureLocalStore() static StorePath useDeriver(const StorePath & path) { - if (path.isDerivation()) return path.clone(); + if (path.isDerivation()) return path; auto info = store->queryPathInfo(path); if (!info->deriver) throw Error("deriver of path '%s' is not known", store->printStorePath(path)); - return info->deriver->clone(); + return *info->deriver; } @@ -214,15 +214,15 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise) { - if (forceRealise) realisePath(storePath); + if (forceRealise) realisePath({storePath}); if (useOutput && storePath.isDerivation()) { auto drv = store->derivationFromPath(storePath); StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path.clone()); + outputs.insert(i.second.path); return outputs; } - else return singleton(storePath.clone()); + else return {storePath}; } @@ -232,7 +232,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, static void printTree(const StorePath & path, const string & firstPad, const string & tailPad, StorePathSet & done) { - if (!done.insert(path.clone()).second) { + if (!done.insert(path).second) { cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } @@ -310,7 +310,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qOutputs: { for (auto & i : opArgs) { auto i2 = store->followLinksToStorePath(i); - if (forceRealise) realisePath(i2); + if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) cout << fmt("%1%\n", store->printStorePath(j.second.path)); @@ -329,13 +329,13 @@ static void opQuery(Strings opFlags, Strings opArgs) if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { for (auto & p : store->queryPathInfo(j)->references) - paths.insert(p.clone()); + paths.insert(p); } else if (query == qReferrers) { StorePathSet tmp; store->queryReferrers(j, tmp); for (auto & i : tmp) - paths.insert(i.clone()); + paths.insert(i); } else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } @@ -391,7 +391,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet roots; for (auto & i : opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - roots.insert(j.clone()); + roots.insert(j); printDotGraph(ref(store), std::move(roots)); break; } @@ -400,7 +400,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet roots; for (auto & i : opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - roots.insert(j.clone()); + roots.insert(j); printGraphML(ref(store), std::move(roots)); break; } @@ -415,7 +415,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet args; for (auto & i : opArgs) for (auto & p : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - args.insert(p.clone()); + args.insert(p); StorePathSet referrers; store->computeFSClosure( @@ -482,10 +482,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opArgs.empty()) { for (auto & i : opArgs) - cout << store->makeValidityRegistration(singleton(store->followLinksToStorePath(i)), true, true); + cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); } else { for (auto & i : store->queryAllValidPaths()) - cout << store->makeValidityRegistration(singleton(i), true, true); + cout << store->makeValidityRegistration({i}, true, true); } } @@ -586,7 +586,7 @@ static void opGC(Strings opFlags, Strings opArgs) // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) - roots2.emplace(link, target.clone()); + roots2.emplace(link, target); for (auto & [link, target] : roots2) std::cout << link << " -> " << store->printStorePath(target) << "\n"; } @@ -830,7 +830,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.emplace_back(path.clone()); + paths2.push_back({path}); unsigned long long downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, @@ -840,7 +840,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!willSubstitute.empty()) try { std::vector subs; - for (auto & p : willSubstitute) subs.emplace_back(p.clone()); + for (auto & p : willSubstitute) subs.push_back({p}); store->buildPaths(subs); } catch (Error & e) { logWarning(e.info()); @@ -895,7 +895,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector paths; for (auto & s : readStrings(in)) - paths.emplace_back(store->parsePathWithOutputs(s)); + paths.push_back(store->parsePathWithOutputs(s)); getBuildSettings(); diff --git a/src/nix/command.cc b/src/nix/command.cc index 71b027719..d62626c26 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -59,19 +59,19 @@ void StorePathsCommand::run(ref store) if (installables.size()) throw UsageError("'--all' does not expect arguments"); for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p.clone()); + storePaths.push_back(p); } else { for (auto & p : toStorePaths(store, realiseMode, installables)) - storePaths.push_back(p.clone()); + storePaths.push_back(p); if (recursive) { StorePathSet closure; - store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); + store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); storePaths.clear(); for (auto & p : closure) - storePaths.push_back(p.clone()); + storePaths.push_back(p); } } @@ -133,7 +133,7 @@ void MixProfile::updateProfile(const Buildables & buildables) for (auto & output : buildable.outputs) { if (result) throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second.clone(); + result = output.second; } } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index c7c38709d..64099f476 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -94,7 +94,7 @@ struct CmdCopy : StorePathsCommand ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, storePathsToSet(storePaths), + copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 3045d7dc3..05a9b9cd9 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,12 +135,12 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); + drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); /* Build the derivation. */ - store->buildPaths({shellDrvPath2}); + store->buildPaths({{shellDrvPath2}}); assert(store->isValidPath(shellOutPath)); @@ -205,7 +205,7 @@ struct Common : InstallableCommand, MixProfile { auto path = installable->getStorePath(); if (path && hasSuffix(path->to_string(), "-env")) - return path->clone(); + return *path; else { auto drvs = toDerivations(store, {installable}); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 937d69206..708a0dc88 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -102,9 +102,9 @@ struct InstallableStorePath : Installable Buildables toBuildables() override { std::map outputs; - outputs.insert_or_assign("out", storePath.clone()); + outputs.insert_or_assign("out", storePath); Buildable b{ - .drvPath = storePath.isDerivation() ? storePath.clone() : std::optional(), + .drvPath = storePath.isDerivation() ? storePath : std::optional(), .outputs = std::move(outputs) }; Buildables bs; @@ -114,7 +114,7 @@ struct InstallableStorePath : Installable std::optional getStorePath() override { - return storePath.clone(); + return storePath; } }; @@ -141,7 +141,7 @@ struct InstallableValue : Installable for (auto & drv : drvs) { Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(b.drvPath->clone()); + drvPaths.insert(*b.drvPath); auto outputName = drv.queryOutputName(); if (outputName == "") @@ -155,10 +155,10 @@ struct InstallableValue : Installable // Hack to recognize .all: if all drvs have the same drvPath, // merge the buildables. if (drvPaths.size() == 1) { - Buildable b{.drvPath = drvPaths.begin()->clone()}; + Buildable b{.drvPath = *drvPaths.begin()}; for (auto & b2 : res) for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second.clone()); + b.outputs.insert_or_assign(output.first, output.second); Buildables bs; bs.push_back(std::move(b)); return bs; @@ -273,7 +273,7 @@ Buildables build(ref store, RealiseMode mode, pathsToBuild.push_back({*b.drvPath, outputNames}); } else for (auto & output : b.outputs) - pathsToBuild.push_back({output.second.clone()}); + pathsToBuild.push_back({output.second}); buildables.push_back(std::move(b)); } } @@ -293,7 +293,7 @@ StorePathSet toStorePaths(ref store, RealiseMode mode, for (auto & b : build(store, mode, installables)) for (auto & output : b.outputs) - outPaths.insert(output.second.clone()); + outPaths.insert(output.second); return outPaths; } @@ -306,7 +306,7 @@ StorePath toStorePath(ref store, RealiseMode mode, if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); - return paths.begin()->clone(); + return *paths.begin(); } StorePathSet toDerivations(ref store, @@ -324,10 +324,10 @@ StorePathSet toDerivations(ref store, if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? - drvPaths.insert(derivers.begin()->clone()); + drvPaths.insert(*derivers.begin()); } } else - drvPaths.insert(b.drvPath->clone()); + drvPaths.insert(*b.drvPath); } return drvPaths; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 1211dad7b..0ebb8f13b 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -36,7 +36,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON void run(ref store, StorePaths storePaths) override { - auto paths = store->topoSortPaths(storePathsToSet(storePaths)); + auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); std::reverse(paths.begin(), paths.end()); @@ -62,7 +62,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON hasSelfReference = true; else { auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second.clone() : ref.clone(); + auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); @@ -79,7 +79,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path.clone()); + if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 88d7fffd4..fb7bacc4c 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -90,7 +90,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON JSONPlaceholder jsonRoot(std::cout); store->pathInfoToJSON(jsonRoot, // FIXME: preserve order? - storePathsToSet(storePaths), + StorePathSet(storePaths.begin(), storePaths.end()), true, showClosureSize, SRI, AllowInvalid); } diff --git a/src/nix/run.cc b/src/nix/run.cc index c9b69aec7..321ee1d11 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -111,16 +111,16 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment std::unordered_set done; std::queue todo; - for (auto & path : outPaths) todo.push(path.clone()); + for (auto & path : outPaths) todo.push(path); setEnviron(); auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { - auto path = todo.front().clone(); + auto path = todo.front(); todo.pop(); - if (!done.insert(path.clone()).second) continue; + if (!done.insert(path).second) continue; if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a4ee2d971..167c974ee 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -106,16 +106,16 @@ struct CmdWhyDepends : SourceExprCommand std::map graph; for (auto & path : closure) - graph.emplace(path.clone(), Node { .path = path.clone(), .refs = cloneStorePathSet(store->queryPathInfo(path)->references) }); + graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references }); // Transpose the graph. for (auto & node : graph) for (auto & ref : node.second.refs) - graph.find(ref)->second.rrefs.insert(node.first.clone()); + graph.find(ref)->second.rrefs.insert(node.first); /* Run Dijkstra's shortest path algorithm to get the distance of every path in the closure to 'dependency'. */ - graph.emplace(dependencyPath.clone(), Node { .path = dependencyPath.clone(), .dist = 0 }); + graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 }); std::priority_queue queue;