From 10d33452e289ded93e192c7d99c1da08a52448e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Tue, 17 Apr 2018 09:13:01 +0200 Subject: [PATCH 0001/2548] =?UTF-8?q?nix-lang=20parser:=C2=A0Add=20mixed?= =?UTF-8?q?=20nested=20attrs=20tests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of issue #2077 --- tests/lang/parse-fail-mixed-nested-attrs.nix | 4 ++++ tests/lang/parse-okay-mixed-nested-attrs-1.nix | 4 ++++ tests/lang/parse-okay-mixed-nested-attrs-2.nix | 4 ++++ tests/lang/parse-okay-nested-attrs.nix | 4 ++++ 4 files changed, 16 insertions(+) create mode 100644 tests/lang/parse-fail-mixed-nested-attrs.nix create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-1.nix create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-2.nix create mode 100644 tests/lang/parse-okay-nested-attrs.nix diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs.nix new file mode 100644 index 000000000..458a7c545 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.z = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 000000000..fd1001c8c --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 000000000..ad066b680 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix new file mode 100644 index 000000000..fd1001c8c --- /dev/null +++ b/tests/lang/parse-okay-nested-attrs.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} From 00584bb0915ee21fb01e9390a901161bdd988197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Wed, 18 Apr 2018 18:39:40 +0200 Subject: [PATCH 0002/2548] parser: Allow mixed nested and top-level attrpaths Fixes #2077. --- src/libexpr/parser.y | 26 ++++++++++++++++--- ...nix => parse-fail-mixed-nested-attrs1.nix} | 2 +- tests/lang/parse-fail-mixed-nested-attrs2.nix | 4 +++ ...attrs-6.nix => parse-okay-dup-attrs-6.nix} | 0 tests/lang/parse-okay-nested-attrs.nix | 4 --- 5 files changed, 28 insertions(+), 8 deletions(-) rename tests/lang/{parse-fail-mixed-nested-attrs.nix => parse-fail-mixed-nested-attrs1.nix} (100%) create mode 100644 tests/lang/parse-fail-mixed-nested-attrs2.nix rename tests/lang/{parse-fail-dup-attrs-6.nix => parse-okay-dup-attrs-6.nix} (100%) delete mode 100644 tests/lang/parse-okay-nested-attrs.nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index ef11dd609..df4fdf032 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -83,7 +83,9 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { @@ -104,11 +106,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } - if (i->symbol.set()) { + // Expr insertion. + // ========================== + if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + ExprAttrs* ae = dynamic_cast(e); + ExprAttrs* jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto ad: ae->attrs) { + ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } diff --git a/tests/lang/parse-fail-mixed-nested-attrs.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs.nix rename to tests/lang/parse-fail-mixed-nested-attrs1.nix index 458a7c545..11e40e66f 100644 --- a/tests/lang/parse-fail-mixed-nested-attrs.nix +++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -1,4 +1,4 @@ { - x = { y = 3; z = 3; }; x.z = 3; + x = { y = 3; z = 3; }; } diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 000000000..17da82e5f --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-6.nix rename to tests/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-nested-attrs.nix b/tests/lang/parse-okay-nested-attrs.nix deleted file mode 100644 index fd1001c8c..000000000 --- a/tests/lang/parse-okay-nested-attrs.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ - x = { y = 3; z = 3; }; - x.q = 3; -} From b2f3a7411a509d5df5a189066870a0f02b6523a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Tue, 1 May 2018 14:42:34 +0200 Subject: [PATCH 0003/2548] nix-lang: Add deep nested mixed attrs test case. --- tests/lang/parse-okay-mixed-nested-attrs-3.nix | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tests/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 000000000..45a33e480 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} From 41f38fbb4b89dc5df244a07410724beca2e14d96 Mon Sep 17 00:00:00 2001 From: Nathan van Doorn Date: Mon, 16 Jul 2018 10:00:42 +0100 Subject: [PATCH 0004/2548] nix-channel documentation: don't suggest deprecated function Running `nix-instantiate --eval -E '(import {}).lib.nixpkgsVersion` emits a warning ``` trace: `lib.nixpkgsVersion` is deprecated, use `lib.version` instead! ``` --- doc/manual/command-ref/nix-channel.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a76..f5159165a 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -112,13 +112,13 @@ $ nix-env -iA nixpkgs.hello You can revert channel updates using : -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.526.dbadfad" From 58a85fa4621faaa89286065f5583328783085722 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 8 Aug 2018 21:21:21 +0200 Subject: [PATCH 0005/2548] mention nix-store --query --roots when a path cannot be deleted --- src/libstore/gc.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b415d5421..65220a9f6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -791,7 +791,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) assertStorePath(i); tryToDelete(state, i); if (state.dead.find(i) == state.dead.end()) - throw Error(format("cannot delete path '%1%' since it is still alive") % i); + throw Error(format( + "cannot delete path '%1%' since it is still alive. " + "To find out why use: " + "nix-store --query --roots" + ) % i); } } else if (options.maxFreed > 0) { From 7e35e914c1aa24957107c666c76f1d834ebae90a Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Fri, 14 Dec 2018 20:07:23 +0100 Subject: [PATCH 0006/2548] fetchGit: allow fetching explicit refs Trying to fetch refs that are not in refs/heads currently fails because it looks for refs/heads/refs/foo instead of refs/foo. eg. builtins.fetchGit { url = https://github.com/NixOS/nixpkgs.git; ref = "refs/pull/1024/head; } --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258..588b0fa4d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -94,7 +94,11 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", "--bare", cacheDir }); } - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); From 82f054d7d5fc8f9de45afa7107557644d1514c98 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 20 Dec 2018 20:05:14 +0100 Subject: [PATCH 0007/2548] installer: update macOS version check to 10.12.2 Nixpkgs will drop support for <10.12 soon and thus a nix release built using the 19.03 channel will also require a newer version of macOS. --- scripts/install-nix-from-closure.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index ab20774bb..e635dcab3 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then exit 1 fi -# macOS support for 10.10 or higher +# macOS support for 10.12.6 or higher if [ "$(uname -s)" = "Darwin" ]; then - if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then - echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi fi From 087be7281a02a6d0e548ae56b910771837023d21 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 13 Jan 2019 10:33:15 -0500 Subject: [PATCH 0008/2548] Treat plain derivation paths in context as normal paths. Previously, plain derivation paths in the string context (e.g. those that arose from builtins.storePath on a drv file, not those that arose from accessing .drvPath of a derivation) were treated somewhat like derivaiton paths derived from .drvPath, except their dependencies weren't recursively added to the input set. With this change, such plain derivation paths are simply treated as paths and added to the source inputs set accordingly, simplifying context handling code and removing the inconsistency. If drvPath-like behavior is desired, the .drv file can be imported and then .drvPath can be accessed. This is a backwards-incompatibility, but storePath is never used on drv files within nixpkgs and almost never used elsewhere. --- .gitignore | 3 +++ src/libexpr/primops.cc | 20 +++----------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 0f2f3ddee..b75c5d489 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ perl/Makefile.config /tests/common.sh /tests/dummy /tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv # /tests/lang/ /tests/lang/*.out diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 60698f740..113ceaf29 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -687,21 +687,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - /* Handle derivation outputs of the form ‘!!’. */ else if (path.at(0) == '!') { std::pair ctx = decodeContext(path); drv.inputDrvs[ctx.first].insert(ctx.second); } - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); - /* Otherwise it's a source file. */ else drv.inputSrcs.insert(path); @@ -1004,13 +995,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu PathSet refs; for (auto path : context) { - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) { - /* See prim_unsafeDiscardOutputDependency. */ - if (path.at(0) != '~') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - path = string(path, 1); - } + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); refs.insert(path); } @@ -1823,7 +1809,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po PathSet context2; for (auto & p : context) - context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); + context2.insert(p.at(0) == '=' ? string(p, 1) : p); mkString(v, s, context2); } From 1d757292d0cb78beec32fcdfe15c2944a4bc4a95 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 13 Jan 2019 12:43:46 -0500 Subject: [PATCH 0009/2548] Add builtins.getContext. This can be very helpful when debugging, as well as enabling complex black magic like surgically removing a single dependency from a string's context. --- src/libexpr/eval.hh | 3 + src/libexpr/primops.cc | 38 ----- src/libexpr/primops/context.cc | 132 ++++++++++++++++++ .../lang/eval-okay-context-introspection.exp | 1 + .../lang/eval-okay-context-introspection.nix | 22 +++ 5 files changed, 158 insertions(+), 38 deletions(-) create mode 100644 src/libexpr/primops/context.cc create mode 100644 tests/lang/eval-okay-context-introspection.exp create mode 100644 tests/lang/eval-okay-context-introspection.nix diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d0f298e16..9fe387891 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -316,6 +316,9 @@ private: /* Return a string representing the type of the value `v'. */ string showType(const Value & v); +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(const string & s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 113ceaf29..0da9f702f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1780,41 +1780,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args } -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); -} - - -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? string(p, 1) : p); - - mkString(v, s, context2); -} - - /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2285,9 +2250,6 @@ void EvalState::createBaseEnv() addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hasContext", 1, prim_hasContext); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); addPrimOp("__split", 2, prim_split); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc new file mode 100644 index 000000000..6849aa261 --- /dev/null +++ b/src/libexpr/primops/context.cc @@ -0,0 +1,132 @@ +#include "primops.hh" +#include "eval-inline.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present, but at least one must be set to something other + than the default. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +} diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp new file mode 100644 index 000000000..27ba77dda --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix new file mode 100644 index 000000000..d9b2ea354 --- /dev/null +++ b/tests/lang/eval-okay-context-introspection.nix @@ -0,0 +1,22 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./eval-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; +in builtins.getContext legit-context == desired-context From 5f1891b795c4c4daac5336d1ec60c94f521ede1d Mon Sep 17 00:00:00 2001 From: Spencer Baugh Date: Wed, 12 Dec 2018 14:19:54 -0500 Subject: [PATCH 0010/2548] nix-store: make --dump-db take a list of paths to dump Inside a derivation, exportReferencesGraph already provides a way to dump the Nix database for a specific closure. On the command line, --dump-db gave us the same information, but only for the entire Nix database at once. With this change, one can now pass a list of paths to --dump-db to get the Nix database dumped for just those paths. (The user is responsible for ensuring this is a closure, like for --export). Among other things, this is useful for deploying a closure to a new host without using --import/--export; one can use tar to transfer the store paths, and --dump-db/--load-db to transfer the validity information. This is useful if the new host doesn't actually have Nix yet, and the closure that is being deployed itself contains Nix. --- doc/manual/command-ref/nix-store.xml | 8 ++++++++ src/nix-store/nix-store.cc | 15 ++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 41a04f265..d73cb92ee 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1282,6 +1282,7 @@ ktorrent-2.2.1/NEWS nix-store + paths @@ -1292,6 +1293,13 @@ Nix database to standard output. It can be loaded into an empty Nix store using . This is useful for making backups and when migrating to different database schemas. +By default, will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +, the user is responsible for passing all the +store paths for a closure. See for an +example. + diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a9ad14762..33138baff 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -485,11 +485,16 @@ static void opReadLog(Strings opFlags, Strings opArgs) static void opDumpDB(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + if (!opArgs.empty()) { + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + for (auto & i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } } From b30be6b450f872f8be6dc8afa28f4b030fa8d1d1 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 14 Jan 2019 11:34:54 -0500 Subject: [PATCH 0011/2548] Add builtins.appendContext. A partner of builtins.getContext, useful for the same reasons. --- src/libexpr/primops/context.cc | 59 ++++++++++++++++++- .../lang/eval-okay-context-introspection.nix | 6 +- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 6849aa261..2d79739ea 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "derivations.hh" namespace nix { @@ -61,8 +62,7 @@ static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscar (i.e. the kind of context you get when referencing .outPath of some derivation). Empty list if missing. Note that for a given path any combination of the above attributes - may be present, but at least one must be set to something other - than the default. + may be present. */ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -129,4 +129,59 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + if (!settings.readOnlyMode) + state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + } diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/lang/eval-okay-context-introspection.nix index d9b2ea354..43178bd2e 100644 --- a/tests/lang/eval-okay-context-introspection.nix +++ b/tests/lang/eval-okay-context-introspection.nix @@ -18,5 +18,7 @@ let }; }; - legit-context = "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; -in builtins.getContext legit-context == desired-context + legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + + constructed-context = builtins.getContext (builtins.appendContext "" desired-context); +in legit-context == constructed-context From 01d07b1e92c298f729a73705907b2987da9a4d0c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Feb 2019 10:49:19 +0100 Subject: [PATCH 0012/2548] Revert "Restore parent mount namespace before executing a child process" This reverts commit a0ef21262f4d5652bfb65cfacaec01d89c475a93. This doesn't work in 'nix run' and nix-shell because setns() fails in multithreaded programs, and Boehm GC mark threads are uncancellable. Fixes #2646. --- src/libstore/build.cc | 2 -- src/libstore/local-store.cc | 2 -- src/libstore/ssh.cc | 3 --- src/libutil/util.cc | 24 ------------------------ src/libutil/util.hh | 10 ---------- src/nix-build/nix-build.cc | 4 ++-- src/nix/edit.cc | 5 ----- src/nix/repl.cc | 2 -- src/nix/run.cc | 4 ++-- 9 files changed, 4 insertions(+), 52 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 59abae9b9..47ee8b48f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2193,7 +2193,6 @@ void DerivationGoal::startBuilder() userNamespaceSync.create(); options.allowVfork = false; - options.restoreMountNamespace = false; Pid helper = startProcess([&]() { @@ -2260,7 +2259,6 @@ void DerivationGoal::startBuilder() #endif { options.allowVfork = !buildUser && !drv->isBuiltin(); - options.restoreMountNamespace = false; pid = startProcess([&]() { runChild(); }, options); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5b4e7ca4c..485fdd691 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -366,8 +366,6 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) throw SysError("setting up a private mount namespace"); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index cf133b57c..5e0e44935 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,4 @@ #include "ssh.hh" -#include "affinity.hh" namespace nix { @@ -35,9 +34,7 @@ std::unique_ptr SSHMaster::startCommand(const std::string auto conn = std::make_unique(); conn->sshPid = startProcess([&]() { - restoreAffinity(); restoreSignals(); - restoreMountNamespace(); close(in.writeSide.get()); close(out.readSide.get()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ce50334e1..7eca35577 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -936,8 +936,6 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) throw SysError("setting death signal"); #endif restoreAffinity(); - if (options.restoreMountNamespace) - restoreMountNamespace(); fun(); } catch (std::exception & e) { try { @@ -1506,26 +1504,4 @@ std::unique_ptr createInterruptCallback(std::function return std::unique_ptr(res.release()); } -static AutoCloseFD fdSavedMountNamespace; - -void saveMountNamespace() -{ -#if __linux__ - std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); -#endif -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d67bddc13..bda87bee4 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -250,7 +250,6 @@ struct ProcessOptions bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = true; - bool restoreMountNamespace = true; }; pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); @@ -515,13 +514,4 @@ typedef std::function PathFilter; extern PathFilter defaultPathFilter; -/* Save the current mount namespace. Ignored if called more than - once. */ -void saveMountNamespace(); - -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ -void restoreMountNamespace(); - - } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 11ea3b1f7..618895d38 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -401,6 +401,8 @@ static void _main(int argc, char * * argv) } else env[var.first] = var.second; + restoreAffinity(); + /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't @@ -444,9 +446,7 @@ static void _main(int argc, char * * argv) auto argPtrs = stringsToCharPtrs(args); - restoreAffinity(); restoreSignals(); - restoreMountNamespace(); execvp(shell.c_str(), argPtrs.data()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index d8d5895bd..c9671f76d 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -3,7 +3,6 @@ #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" -#include "affinity.hh" #include @@ -73,10 +72,6 @@ struct CmdEdit : InstallableCommand stopProgressBar(); - restoreAffinity(); - restoreSignals(); - restoreMountNamespace(); - execvp(args.front().c_str(), stringsToCharPtrs(args).data()); throw SysError("cannot run editor '%s'", editor); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index dd3d9ed97..227affc60 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -337,8 +337,6 @@ static int runProgram(const string & program, const Strings & args) if (pid == -1) throw SysError("forking"); if (pid == 0) { restoreAffinity(); - restoreSignals(); - restoreMountNamespace(); execvp(program.c_str(), stringsToCharPtrs(args2).data()); _exit(1); } diff --git a/src/nix/run.cc b/src/nix/run.cc index 129707298..35b763345 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -153,9 +153,9 @@ struct CmdRun : InstallablesCommand stopProgressBar(); - restoreAffinity(); restoreSignals(); - restoreMountNamespace(); + + restoreAffinity(); /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location From 8ac1130cc2f0d717fa49adf18465ecf5613d295c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 9 Feb 2019 20:32:12 +0100 Subject: [PATCH 0013/2548] nix-daemon: add variable to disable fork safety Since macOS 10.14 this has become an error, causing problems if the nix-daemon loads nix during substitution (this is a forked process). Workaround for #2523. --- misc/launchd/org.nixos.nix-daemon.plist.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 549619a57..92ed12fa5 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,6 +2,11 @@ + EnvironmentVariables + + OBJC_DISABLE_INITIALIZE_FORK_SAFETY + YES + Label org.nixos.nix-daemon KeepAlive From aa0e2a2e70a3519a9dcb9b1da000a13c01aa6cc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Nov 2018 16:28:43 +0100 Subject: [PATCH 0014/2548] Make constant primops lazy --- src/libexpr/eval.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a194d0e0..3891e8666 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -439,14 +439,21 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { + auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ if (arity == 0) { + auto vPrimOp = allocValue(); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = new PrimOp(primOp, 1, sym); Value v; - primOp(*this, noPos, nullptr, v); + mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } + Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); v->type = tPrimOp; v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; From 15a16e5c05d547ec07170df2392263e5e891447b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Nov 2018 15:59:52 +0100 Subject: [PATCH 0015/2548] MultiCommand: Simplify construction --- src/nix/command.cc | 8 +++++--- src/nix/command.hh | 8 ++++---- src/nix/main.cc | 7 ++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 3d7d582d6..e760c17d5 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -4,7 +4,7 @@ namespace nix { -Commands * RegisterCommand::commands = 0; +std::vector> * RegisterCommand::commands = 0; void Command::printHelp(const string & programName, std::ostream & out) { @@ -21,9 +21,11 @@ void Command::printHelp(const string & programName, std::ostream & out) } } -MultiCommand::MultiCommand(const Commands & _commands) - : commands(_commands) +MultiCommand::MultiCommand(const std::vector> & _commands) { + for (auto & command : _commands) + commands.emplace(command->name(), command); + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { assert(!command); auto i = commands.find(ss[0]); diff --git a/src/nix/command.hh b/src/nix/command.hh index 97a6fee7f..2108aa674 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -173,7 +173,7 @@ public: std::shared_ptr command; - MultiCommand(const Commands & commands); + MultiCommand(const std::vector> & commands); void printHelp(const string & programName, std::ostream & out) override; @@ -185,12 +185,12 @@ public: /* A helper class for registering commands globally. */ struct RegisterCommand { - static Commands * commands; + static std::vector> * commands; RegisterCommand(ref command) { - if (!commands) commands = new Commands; - commands->emplace(command->name(), command); + if (!commands) commands = new std::vector>; + commands->push_back(command); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 64c1dc357..3d4348d28 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -57,10 +57,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs "--help-config' for a list of configuration settings.\n"; } + void printHelp(const string & programName, std::ostream & out) + { + MultiCommand::printHelp(programName, out); + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + } + void showHelpAndExit() { printHelp(programName, std::cout); - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; throw Exit(); } }; From f70434b1fbbdb0e188718f0c55a8156a7aa08744 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Nov 2018 16:03:31 +0100 Subject: [PATCH 0016/2548] Move Command and MultiCommand to libutil --- src/libutil/args.cc | 69 ++++++++++++++++++++++++++++++++++++++++++ src/libutil/args.hh | 41 +++++++++++++++++++++++++ src/nix/command.cc | 74 --------------------------------------------- src/nix/command.hh | 41 ------------------------- src/nix/main.cc | 5 +++ 5 files changed, 115 insertions(+), 115 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7af2a1bf7..2837dacc9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -200,4 +200,73 @@ void printTable(std::ostream & out, const Table2 & table) } } +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + +MultiCommand::MultiCommand(const std::vector> & _commands) +{ + for (auto & command : _commands) + commands.emplace(command->name(), command); + + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { + assert(!command); + auto i = commands.find(ss[0]); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", ss[0]); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " ... ...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) { + auto descr = command.second->description(); + if (!descr.empty()) + table.push_back(std::make_pair(command.second->name(), descr)); + } + printTable(out, table); +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ad5fcca39..bf69bf4b6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -188,6 +188,47 @@ public: friend class MultiCommand; }; +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const string & programName, std::ostream & out) override; +}; + +typedef std::map> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘ ’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::shared_ptr command; + + MultiCommand(const std::vector> & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + Strings argvToStrings(int argc, char * * argv); /* Helper function for rendering argument labels. */ diff --git a/src/nix/command.cc b/src/nix/command.cc index e760c17d5..5967ab36c 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -6,80 +6,6 @@ namespace nix { std::vector> * RegisterCommand::commands = 0; -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } -} - -MultiCommand::MultiCommand(const std::vector> & _commands) -{ - for (auto & command : _commands) - commands.emplace(command->name(), command); - - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { - assert(!command); - auto i = commands.find(ss[0]); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second; - }}); -} - -void MultiCommand::printHelp(const string & programName, std::ostream & out) -{ - if (command) { - command->printHelp(programName + " " + command->name(), out); - return; - } - - out << "Usage: " << programName << " ... ...\n"; - - out << "\n"; - out << "Common flags:\n"; - printFlags(out); - - out << "\n"; - out << "Available commands:\n"; - - Table2 table; - for (auto & command : commands) { - auto descr = command.second->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command.second->name(), descr)); - } - printTable(out, table); - -#if 0 - out << "\n"; - out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-'.\n"; -#endif -} - -bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) -{ - if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; - return false; -} - -bool MultiCommand::processArgs(const Strings & args, bool finish) -{ - if (command) - return command->processArgs(args, finish); - else - return Args::processArgs(args, finish); -} - StoreCommand::StoreCommand() { } diff --git a/src/nix/command.hh b/src/nix/command.hh index 2108aa674..04183c7ed 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -11,27 +11,6 @@ struct Value; class Bindings; class EvalState; -/* A command is an argument parser that can be executed by calling its - run() method. */ -struct Command : virtual Args -{ - virtual std::string name() = 0; - virtual void prepare() { }; - virtual void run() = 0; - - struct Example - { - std::string description; - std::string command; - }; - - typedef std::list Examples; - - virtual Examples examples() { return Examples(); } - - void printHelp(const string & programName, std::ostream & out) override; -}; - class Store; /* A command that require a Nix store. */ @@ -162,26 +141,6 @@ struct StorePathCommand : public InstallablesCommand void run(ref store) override; }; -typedef std::map> Commands; - -/* An argument parser that supports multiple subcommands, - i.e. ‘ ’. */ -class MultiCommand : virtual Args -{ -public: - Commands commands; - - std::shared_ptr command; - - MultiCommand(const std::vector> & commands); - - void printHelp(const string & programName, std::ostream & out) override; - - bool processFlag(Strings::iterator & pos, Strings::iterator end) override; - - bool processArgs(const Strings & args, bool finish) override; -}; - /* A helper class for registering commands globally. */ struct RegisterCommand { diff --git a/src/nix/main.cc b/src/nix/main.cc index 3d4348d28..4b909736d 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -60,6 +60,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void printHelp(const string & programName, std::ostream & out) { MultiCommand::printHelp(programName, out); + +#if 0 + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-'.\n"; +#endif + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; } From c02da997570ac0d9b595d787bea8cb5a4e3cc1f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Nov 2018 19:55:48 +0100 Subject: [PATCH 0017/2548] EvalState::allocAttr(): Add convenience method --- src/libexpr/attr-set.cc | 6 ++++++ src/libexpr/eval.hh | 1 + 2 files changed, 7 insertions(+) diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 0785897d2..b1d61a285 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d0f298e16..60cf0f87f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -264,6 +264,7 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); Bindings * allocBindings(size_t capacity); From f216c76c56cdffb5214d074a7d44812843dd174f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Nov 2018 19:57:20 +0100 Subject: [PATCH 0018/2548] Bindings::get(): Add convenience method This allows writing attribute lookups as if (auto name = value.attrs->get(state.sName)) ... --- configure.ac | 2 +- src/libexpr/attr-set.hh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5a2526672..b550231cb 100644 --- a/configure.ac +++ b/configure.ac @@ -62,7 +62,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_14 +AX_CXX_COMPILE_STDCXX_17 # Use 64-bit file system calls so that we can support files > 2 GiB. diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 3119a1848..6c5fb21ad 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -4,6 +4,7 @@ #include "symbol-table.hh" #include +#include namespace nix { @@ -63,6 +64,14 @@ public: return end(); } + std::optional get(const Symbol & name) + { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return &*i; + return {}; + } + iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } From 7a5cf31060289de61370643937277b5d0d5d178c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Nov 2018 19:18:36 +0100 Subject: [PATCH 0019/2548] Initial flake support --- corepkgs/default-installation-source.nix | 3 + corepkgs/local.mk | 8 +- src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 20 ++- src/libexpr/primops/fetchGit.cc | 9 +- src/libexpr/primops/fetchGit.hh | 23 ++++ src/libexpr/primops/flake.cc | 161 +++++++++++++++++++++++ src/nix/flake.cc | 65 +++++++++ src/nix/installables.cc | 39 +----- 9 files changed, 282 insertions(+), 47 deletions(-) create mode 100644 corepkgs/default-installation-source.nix create mode 100644 src/libexpr/primops/fetchGit.hh create mode 100644 src/libexpr/primops/flake.cc create mode 100644 src/nix/flake.cc diff --git a/corepkgs/default-installation-source.nix b/corepkgs/default-installation-source.nix new file mode 100644 index 000000000..71ba04452 --- /dev/null +++ b/corepkgs/default-installation-source.nix @@ -0,0 +1,3 @@ +builtins.mapAttrs (flakeName: flakeInfo: + (getFlake flakeInfo.uri).${flakeName}.provides.packages or {}) + builtins.flakeRegistry diff --git a/corepkgs/local.mk b/corepkgs/local.mk index 362c8eb61..41aaec63b 100644 --- a/corepkgs/local.mk +++ b/corepkgs/local.mk @@ -1,4 +1,10 @@ -corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix +corepkgs_FILES = \ + buildenv.nix \ + unpack-channel.nix \ + derivation.nix \ + fetchurl.nix \ + imported-drv-to-derivation.nix \ + default-installation-source.nix $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3891e8666..e3a264277 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -290,6 +290,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) + , sDescription(symbols.create("description")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 60cf0f87f..674b08f45 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -72,7 +72,8 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sOutputHash, sOutputHashAlgo, sOutputHashMode; + sOutputHash, sOutputHashAlgo, sOutputHashMode, + sDescription; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -311,6 +312,23 @@ private: friend struct ExprOpConcatLists; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); + +public: + + struct FlakeRegistry + { + struct Entry + { + std::string uri; + }; + std::map entries; + }; + + const FlakeRegistry & getFlakeRegistry(); + +private: + std::unique_ptr _flakeRegistry; + std::once_flag _flakeRegistryInit; }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258..6b6ca08d1 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,3 +1,4 @@ +#include "fetchGit.hh" #include "primops.hh" #include "eval-inline.hh" #include "download.hh" @@ -15,14 +16,6 @@ using namespace std::string_literals; namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh new file mode 100644 index 000000000..23ab2fae9 --- /dev/null +++ b/src/libexpr/primops/fetchGit.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "store-api.hh" + +#include + +namespace nix { + +struct GitInfo +{ + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; +}; + +GitInfo exportGit(ref store, const std::string & uri, + std::experimental::optional ref, std::string rev, + const std::string & name); + +extern std::regex revRegex; + +} diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc new file mode 100644 index 000000000..457c30948 --- /dev/null +++ b/src/libexpr/primops/flake.cc @@ -0,0 +1,161 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "fetchGit.hh" +#include "download.hh" + +#include +#include + +namespace nix { + +const EvalState::FlakeRegistry & EvalState::getFlakeRegistry() +{ + std::call_once(_flakeRegistryInit, [&]() + { + _flakeRegistry = std::make_unique(); + + if (!evalSettings.pureEval) { + + auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; + + auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); + + auto json = nlohmann::json::parse(*registryFile.data); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", registryUri, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry; + entry.uri = i->value("uri", ""); + if (entry.uri.empty()) + throw Error("invalid flake registry entry"); + _flakeRegistry->entries.emplace(i.key(), entry); + } + } + }); + + return *_flakeRegistry; +} + +static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto registry = state.getFlakeRegistry(); + + state.mkAttrs(v, registry.entries.size()); + + for (auto & entry : registry.entries) { + auto vEntry = state.allocAttr(v, entry.first); + state.mkAttrs(*vEntry, 2); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.uri); + vEntry->attrs->sort(); + } + + v.attrs->sort(); +} + +static RegisterPrimOp r1("__flakeRegistry", 0, prim_flakeRegistry); + +struct Flake +{ + std::string name; + std::string description; + Path path; + std::set requires; + Value * vProvides; // FIXME: gc +}; + +static Flake fetchFlake(EvalState & state, const std::string & flakeUri) +{ + Flake flake; + + auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source"); + + state.store->assertStorePath(gitInfo.storePath); + + Value vInfo; + state.evalFile(gitInfo.storePath + "/flake.nix", vInfo); + + state.forceAttrs(vInfo); + + if (auto name = vInfo.attrs->get(state.sName)) + flake.name = state.forceStringNoCtx(*(**name).value, *(**name).pos); + else + throw Error("flake lacks attribute 'name'"); + + if (auto description = vInfo.attrs->get(state.sDescription)) + flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + + if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) { + state.forceList(*(**requires).value, *(**requires).pos); + for (unsigned int n = 0; n < (**requires).value->listSize(); ++n) + flake.requires.insert(state.forceStringNoCtx( + *(**requires).value->listElems()[n], *(**requires).pos)); + } + + if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { + state.forceFunction(*(**provides).value, *(**provides).pos); + flake.vProvides = (**provides).value; + } else + throw Error("flake lacks attribute 'provides'"); + + return flake; +} + +static std::map resolveFlakes(EvalState & state, const StringSet & flakeUris) +{ + auto registry = state.getFlakeRegistry(); + + std::map done; + std::queue todo; + for (auto & i : flakeUris) todo.push(i); + + while (!todo.empty()) { + auto flakeUri = todo.front(); + todo.pop(); + if (done.count(flakeUri)) continue; + + auto flake = fetchFlake(state, flakeUri); + + for (auto & require : flake.requires) { + auto i = registry.entries.find(require); + if (i == registry.entries.end()) + throw Error("unknown flake '%s'", require); + todo.push(i->second.uri); + } + + done.emplace(flake.name, flake); + } + + return done; +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string flakeUri = state.forceStringNoCtx(*args[0], pos); + + auto flakes = resolveFlakes(state, {flakeUri}); + + auto vResult = state.allocValue(); + + state.mkAttrs(*vResult, flakes.size()); + + for (auto & flake : flakes) { + auto vFlake = state.allocAttr(*vResult, flake.second.name); + state.mkAttrs(*vFlake, 2); + mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); + auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); + mkApp(*vProvides, *flake.second.vProvides, *vResult); + vFlake->attrs->sort(); + } + + vResult->attrs->sort(); + + v = *vResult; +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..98cd90c64 --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,65 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "progress-bar.hh" +#include "eval.hh" + +using namespace nix; + +struct CmdFlakeList : StoreCommand, MixEvalArgs +{ + std::string name() override + { + return "list"; + } + + std::string description() override + { + return "list available Nix flakes"; + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + auto registry = evalState->getFlakeRegistry(); + + stopProgressBar(); + + for (auto & entry : registry.entries) { + std::cout << entry.first << " " << entry.second.uri << "\n"; + } + } +}; + +struct CmdFlake : virtual MultiCommand, virtual Command +{ + CmdFlake() + : MultiCommand({make_ref()}) + { + } + + std::string name() override + { + return "flake"; + } + + std::string description() override + { + return "manage Nix flakes"; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + command->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0c1ad3ab3..9b7b96c25 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -26,47 +26,12 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) { if (vSourceExpr) return vSourceExpr; - auto sToplevel = state.symbols.create("_toplevel"); - vSourceExpr = state.allocValue(); if (file != "") state.evalFile(lookupFileArg(state, file), *vSourceExpr); - - else { - - /* Construct the installation source from $NIX_PATH. */ - - auto searchPath = state.getSearchPath(); - - state.mkAttrs(*vSourceExpr, searchPath.size() + 1); - - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); - - std::unordered_set seen; - - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); -#if 0 - auto res = state.resolveSearchPathElem(i); - if (!res.first) continue; - if (!pathExists(res.second)) continue; - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), res.second)); -#endif - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), *v2); - } - - vSourceExpr->attrs->sort(); - } + else + state.evalFile(lookupFileArg(state, ""), *vSourceExpr); return vSourceExpr; } From ef4cf4e681bfe30b15b8c2940b51b322bce5b6d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Nov 2018 16:11:15 +0100 Subject: [PATCH 0020/2548] Introduce flake URIs --- src/libexpr/primops/flake.cc | 48 +++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 457c30948..9dc6fa1f1 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -4,6 +4,7 @@ #include "download.hh" #include +#include #include namespace nix { @@ -67,16 +68,39 @@ struct Flake Value * vProvides; // FIXME: gc }; -static Flake fetchFlake(EvalState & state, const std::string & flakeUri) +std::regex flakeRegex("^flake:([a-zA-Z][a-zA-Z0-9_-]+)$"); + +static Path fetchFlake(EvalState & state, const std::string & flakeUri) { + std::smatch match; + + if (std::regex_match(flakeUri, match, flakeRegex)) { + auto flakeName = match[1]; + auto registry = state.getFlakeRegistry(); + auto i = registry.entries.find(flakeName); + if (i == registry.entries.end()) + throw Error("unknown flake '%s'", flakeName); + return fetchFlake(state, i->second.uri); + } + + else if (hasPrefix(flakeUri, "/") || hasPrefix(flakeUri, "git://")) { + auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source"); + return gitInfo.storePath; + } + + else + throw Error("unsupported flake URI '%s'", flakeUri); +} + +static Flake getFlake(EvalState & state, const std::string & flakeUri) +{ + auto flakePath = fetchFlake(state, flakeUri); + state.store->assertStorePath(flakePath); + Flake flake; - auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source"); - - state.store->assertStorePath(gitInfo.storePath); - Value vInfo; - state.evalFile(gitInfo.storePath + "/flake.nix", vInfo); + state.evalFile(flakePath + "/flake.nix", vInfo); state.forceAttrs(vInfo); @@ -106,8 +130,6 @@ static Flake fetchFlake(EvalState & state, const std::string & flakeUri) static std::map resolveFlakes(EvalState & state, const StringSet & flakeUris) { - auto registry = state.getFlakeRegistry(); - std::map done; std::queue todo; for (auto & i : flakeUris) todo.push(i); @@ -117,14 +139,10 @@ static std::map resolveFlakes(EvalState & state, const Strin todo.pop(); if (done.count(flakeUri)) continue; - auto flake = fetchFlake(state, flakeUri); + auto flake = getFlake(state, flakeUri); - for (auto & require : flake.requires) { - auto i = registry.entries.find(require); - if (i == registry.entries.end()) - throw Error("unknown flake '%s'", require); - todo.push(i->second.uri); - } + for (auto & require : flake.requires) + todo.push(require); done.emplace(flake.name, flake); } From dcae46ab146b735aa49fcf4cad4a320e79362c5e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Dec 2018 13:20:59 +0100 Subject: [PATCH 0021/2548] Add github URIs For example, github:edolstra/dwarffs is more-or-less equivalent to https://github.com/edolstra/dwarffs.git. It's a much faster way to get GitHub repositories: it fetches tarballs rather than entire Git repositories. It also allows fetching specific revisions by hash without specifying a ref (e.g. a branch name): github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 --- src/libexpr/primops/flake.cc | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 9dc6fa1f1..81e912402 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -68,7 +68,8 @@ struct Flake Value * vProvides; // FIXME: gc }; -std::regex flakeRegex("^flake:([a-zA-Z][a-zA-Z0-9_-]+)$"); +std::regex flakeRegex("^flake:([a-zA-Z][a-zA-Z0-9_-]*)(/[a-zA-Z][a-zA-Z0-9_.-]*)?$"); +std::regex githubRegex("^github:([a-zA-Z][a-zA-Z0-9_-]*)/([a-zA-Z][a-zA-Z0-9_-]*)(/([a-zA-Z][a-zA-Z0-9_-]*))?$"); static Path fetchFlake(EvalState & state, const std::string & flakeUri) { @@ -76,6 +77,7 @@ static Path fetchFlake(EvalState & state, const std::string & flakeUri) if (std::regex_match(flakeUri, match, flakeRegex)) { auto flakeName = match[1]; + auto revOrRef = match[2]; auto registry = state.getFlakeRegistry(); auto i = registry.entries.find(flakeName); if (i == registry.entries.end()) @@ -83,6 +85,25 @@ static Path fetchFlake(EvalState & state, const std::string & flakeUri) return fetchFlake(state, i->second.uri); } + else if (std::regex_match(flakeUri, match, githubRegex)) { + auto owner = match[1]; + auto repo = match[2]; + auto revOrRef = match[4].str(); + if (revOrRef.empty()) revOrRef = "master"; + + // FIXME: require hash in pure mode. + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + auto storePath = getDownloader()->downloadCached(state.store, + fmt("https://api.github.com/repos/%s/%s/tarball/%s", owner, repo, revOrRef), + true, "source"); + + // FIXME: extract revision hash from ETag. + + return storePath; + } + else if (hasPrefix(flakeUri, "/") || hasPrefix(flakeUri, "git://")) { auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source"); return gitInfo.storePath; From 52419f8db3ed4806f8114a98f94a68f3b249f065 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Feb 2019 13:01:39 +0100 Subject: [PATCH 0022/2548] Add flake registry This will eventually be moved to nixos.org. --- flake-registry.json | 11 +++++++++++ local.mk | 2 ++ src/libexpr/primops/flake.cc | 8 ++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 flake-registry.json diff --git a/flake-registry.json b/flake-registry.json new file mode 100644 index 000000000..b850daa74 --- /dev/null +++ b/flake-registry.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "flakes": { + "dwarffs": { + "uri": "github:edolstra/dwarffs/flake" + }, + "nixpkgs": { + "uri": "github:edolstra/nixpkgs/flake" + } + } +} diff --git a/local.mk b/local.mk index 4b380176f..11ed9c0a6 100644 --- a/local.mk +++ b/local.mk @@ -10,3 +10,5 @@ GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) + +$(eval $(call install-data-in,$(d)/flake-registry.json,$(datadir)/nix)) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 81e912402..1367fa420 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -17,15 +17,19 @@ const EvalState::FlakeRegistry & EvalState::getFlakeRegistry() if (!evalSettings.pureEval) { +#if 0 auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); +#endif - auto json = nlohmann::json::parse(*registryFile.data); + auto registryFile = readFile(settings.nixDataDir + "/nix/flake-registry.json"); + + auto json = nlohmann::json::parse(registryFile); auto version = json.value("version", 0); if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", registryUri, version); + throw Error("flake registry '%s' has unsupported version %d", registryFile, version); auto flakes = json["flakes"]; for (auto i = flakes.begin(); i != flakes.end(); ++i) { From c8a0b9d5cbfe6619f8b38118f5b1d1875d1c5309 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:43:32 +0100 Subject: [PATCH 0023/2548] experimental/optional -> optional --- src/libexpr/eval.hh | 2 +- src/libexpr/get-drvs.cc | 2 +- src/libexpr/get-drvs.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchGit.cc | 4 ++-- src/libexpr/primops/fetchGit.hh | 2 +- src/libstore/build.cc | 10 +++++----- src/libstore/parsed-derivations.cc | 4 ++-- src/libstore/parsed-derivations.hh | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libutil/lru-cache.hh | 4 ++-- src/libutil/serialise.cc | 2 +- src/libutil/util.cc | 2 +- src/libutil/util.hh | 6 +++--- src/nix/add-to-store.cc | 2 +- 15 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 674b08f45..c8ee63551 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -82,7 +82,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional allowedPaths; + std::optional allowedPaths; Value vEmptySet; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3..21a4d7917 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional getDerivation(EvalState & state, Value & v, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index daaa635fe..d7860fc6a 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -78,7 +78,7 @@ typedef list DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional getDerivation(EvalState & state, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 60698f740..f787ad96b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -555,7 +555,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional outputHash; + std::optional outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 6b6ca08d1..3027e0f2d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -19,7 +19,7 @@ namespace nix { std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, - std::experimental::optional ref, std::string rev, + std::optional ref, std::string rev, const std::string & name) { if (evalSettings.pureEval && rev == "") @@ -183,7 +183,7 @@ GitInfo exportGit(ref store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional ref; + std::optional ref; std::string rev; std::string name = "source"; PathSet context; diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 23ab2fae9..6031e09e1 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -15,7 +15,7 @@ struct GitInfo }; GitInfo exportGit(ref store, const std::string & uri, - std::experimental::optional ref, std::string rev, + std::optional ref, std::string rev, const std::string & name); extern std::regex revRegex; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 47ee8b48f..6b88b1307 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2413,7 +2413,7 @@ void DerivationGoal::writeStructuredAttrs() objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional { + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) return shellEscape(value); @@ -3311,8 +3311,8 @@ void DerivationGoal::checkOutputs(const std::map & outputs) struct Checks { bool ignoreSelfRefs = false; - std::experimental::optional maxSize, maxClosureSize; - std::experimental::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; }; /* Compute the closure and closure size of some output. This @@ -3359,7 +3359,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) info.path, closureSize, *checks.maxClosureSize); } - auto checkRefs = [&](const std::experimental::optional & value, bool allowed, bool recursive) + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) { if (!value) return; @@ -3413,7 +3413,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) if (maxClosureSize != output->end()) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::experimental::optional { + auto get = [&](const std::string & name) -> std::optional { auto i = output->find(name); if (i != output->end()) { Strings res; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index dc3286482..17fde00a0 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) } } -std::experimental::optional ParsedDerivation::getStringAttr(const std::string & name) const +std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); @@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const } } -std::experimental::optional ParsedDerivation::getStringsAttr(const std::string & name) const +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0a82c1461..ed07dc652 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -8,22 +8,22 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::experimental::optional structuredAttrs; + std::optional structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::experimental::optional & getStructuredAttrs() const + const std::optional & getStructuredAttrs() const { return structuredAttrs; } - std::experimental::optional getStringAttr(const std::string & name) const; + std::optional getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string & name, bool def = false) const; - std::experimental::optional getStringsAttr(const std::string & name) const; + std::optional getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f554b598..919c6d819 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -149,7 +149,7 @@ public: private: ref openConnection() override; - std::experimental::optional path; + std::optional path; }; diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e63..8b83f842c 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include #include -#include +#include namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional get(const Key & key) + std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0e75eeec2..8201549fd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,7 +171,7 @@ std::unique_ptr sinkToSource( std::function fun; std::function eof; - std::experimental::optional coro; + std::optional coro; bool started = false; SinkToSource(std::function fun, std::function eof) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7eca35577..e3dcd246c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -965,7 +965,7 @@ std::vector stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional & input) + const std::optional & input) { RunOptions opts(program, args); opts.searchPath = searchPath; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bda87bee4..9f239bff3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -259,14 +259,14 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional & input = {}); + const std::optional & input = {}); struct RunOptions { Path program; bool searchPath = true; Strings args; - std::experimental::optional input; + std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool _killStderr = false; diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d..e86b96e3f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional namePart; + std::optional namePart; CmdAddToStore() { From 0cd7f2cd8d99071ebfb06a8f0d6a18efed6cd42e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:44:20 +0100 Subject: [PATCH 0024/2548] pkg-config files: Use c++17 --- src/libexpr/nix-expr.pc.in | 2 +- src/libmain/nix-main.pc.in | 2 +- src/libstore/nix-store.pc.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f45..80f7a492b 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c48..37b03dcd4 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faad..6d67b1e03 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 From 91a6a47b0e98f4114c263ef32895e749639c50ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 18:23:11 +0100 Subject: [PATCH 0025/2548] Improve flake references --- src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 10 +- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchGit.hh | 2 - src/libexpr/primops/flake.cc | 107 +++++++++++---------- src/libexpr/primops/flake.hh | 17 ++++ src/libexpr/primops/flakeref.cc | 139 ++++++++++++++++++++++++++++ src/libexpr/primops/flakeref.hh | 158 ++++++++++++++++++++++++++++++++ src/nix/flake.cc | 3 +- 9 files changed, 380 insertions(+), 59 deletions(-) create mode 100644 src/libexpr/primops/flake.hh create mode 100644 src/libexpr/primops/flakeref.cc create mode 100644 src/libexpr/primops/flakeref.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e3a264277..548eef31b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,6 +7,7 @@ #include "eval-inline.hh" #include "download.hh" #include "json.hh" +#include "primops/flake.hh" #include #include diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c8ee63551..35c01b97a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -17,6 +17,7 @@ namespace nix { class Store; class EvalState; enum RepairFlag : bool; +struct FlakeRegistry; typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -315,15 +316,6 @@ private: public: - struct FlakeRegistry - { - struct Entry - { - std::string uri; - }; - std::map entries; - }; - const FlakeRegistry & getFlakeRegistry(); private: diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 3027e0f2d..62e9dfc0e 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -16,7 +16,7 @@ using namespace std::string_literals; namespace nix { -std::regex revRegex("^[0-9a-fA-F]{40}$"); +extern std::regex revRegex; GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 6031e09e1..d7a0e165a 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -18,6 +18,4 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name); -extern std::regex revRegex; - } diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1367fa420..5e92b1da3 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -1,3 +1,4 @@ +#include "flake.hh" #include "primops.hh" #include "eval-inline.hh" #include "fetchGit.hh" @@ -9,7 +10,7 @@ namespace nix { -const EvalState::FlakeRegistry & EvalState::getFlakeRegistry() +const FlakeRegistry & EvalState::getFlakeRegistry() { std::call_once(_flakeRegistryInit, [&]() { @@ -33,10 +34,7 @@ const EvalState::FlakeRegistry & EvalState::getFlakeRegistry() auto flakes = json["flakes"]; for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry; - entry.uri = i->value("uri", ""); - if (entry.uri.empty()) - throw Error("invalid flake registry entry"); + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; _flakeRegistry->entries.emplace(i.key(), entry); } } @@ -54,7 +52,7 @@ static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * arg for (auto & entry : registry.entries) { auto vEntry = state.allocAttr(v, entry.first); state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.uri); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); vEntry->attrs->sort(); } @@ -63,44 +61,53 @@ static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * arg static RegisterPrimOp r1("__flakeRegistry", 0, prim_flakeRegistry); +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef) +{ + if (auto refData = std::get_if(&flakeRef.data)) { + auto registry = state.getFlakeRegistry(); + auto i = registry.entries.find(refData->id); + if (i == registry.entries.end()) + throw Error("cannot find flake '%s' in the flake registry", refData->id); + auto newRef = FlakeRef(i->second.ref); + if (!newRef.isDirect()) + throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); + return newRef; + } else + return flakeRef; +} + struct Flake { - std::string name; + FlakeId id; std::string description; Path path; std::set requires; Value * vProvides; // FIXME: gc + // commit hash + // date + // content hash }; -std::regex flakeRegex("^flake:([a-zA-Z][a-zA-Z0-9_-]*)(/[a-zA-Z][a-zA-Z0-9_.-]*)?$"); -std::regex githubRegex("^github:([a-zA-Z][a-zA-Z0-9_-]*)/([a-zA-Z][a-zA-Z0-9_-]*)(/([a-zA-Z][a-zA-Z0-9_-]*))?$"); - -static Path fetchFlake(EvalState & state, const std::string & flakeUri) +static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) { - std::smatch match; - - if (std::regex_match(flakeUri, match, flakeRegex)) { - auto flakeName = match[1]; - auto revOrRef = match[2]; - auto registry = state.getFlakeRegistry(); - auto i = registry.entries.find(flakeName); - if (i == registry.entries.end()) - throw Error("unknown flake '%s'", flakeName); - return fetchFlake(state, i->second.uri); - } - - else if (std::regex_match(flakeUri, match, githubRegex)) { - auto owner = match[1]; - auto repo = match[2]; - auto revOrRef = match[4].str(); - if (revOrRef.empty()) revOrRef = "master"; + assert(flakeRef.isDirect()); + if (auto refData = std::get_if(&flakeRef.data)) { // FIXME: require hash in pure mode. // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. + + // FIXME: support passing auth tokens for private repos. + auto storePath = getDownloader()->downloadCached(state.store, - fmt("https://api.github.com/repos/%s/%s/tarball/%s", owner, repo, revOrRef), + fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + refData->rev + ? refData->rev->to_string(Base16, false) + : refData->ref + ? *refData->ref + : "master"), true, "source"); // FIXME: extract revision hash from ETag. @@ -108,18 +115,18 @@ static Path fetchFlake(EvalState & state, const std::string & flakeUri) return storePath; } - else if (hasPrefix(flakeUri, "/") || hasPrefix(flakeUri, "git://")) { - auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source"); + else if (auto refData = std::get_if(&flakeRef.data)) { + auto gitInfo = exportGit(state.store, refData->uri, refData->ref, + refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); return gitInfo.storePath; } - else - throw Error("unsupported flake URI '%s'", flakeUri); + else abort(); } -static Flake getFlake(EvalState & state, const std::string & flakeUri) +static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - auto flakePath = fetchFlake(state, flakeUri); + auto flakePath = fetchFlake(state, flakeRef); state.store->assertStorePath(flakePath); Flake flake; @@ -130,7 +137,7 @@ static Flake getFlake(EvalState & state, const std::string & flakeUri) state.forceAttrs(vInfo); if (auto name = vInfo.attrs->get(state.sName)) - flake.name = state.forceStringNoCtx(*(**name).value, *(**name).pos); + flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); else throw Error("flake lacks attribute 'name'"); @@ -153,23 +160,31 @@ static Flake getFlake(EvalState & state, const std::string & flakeUri) return flake; } -static std::map resolveFlakes(EvalState & state, const StringSet & flakeUris) +/* Given a set of flake references, recursively fetch them and their + dependencies. */ +static std::map resolveFlakes(EvalState & state, const std::vector & flakeRefs) { - std::map done; - std::queue todo; - for (auto & i : flakeUris) todo.push(i); + std::map done; + std::queue todo; + for (auto & i : flakeRefs) todo.push(i); while (!todo.empty()) { - auto flakeUri = todo.front(); + auto flakeRef = todo.front(); todo.pop(); - if (done.count(flakeUri)) continue; - auto flake = getFlake(state, flakeUri); + if (auto refData = std::get_if(&flakeRef.data)) { + if (done.count(refData->id)) continue; // optimization + flakeRef = lookupFlake(state, flakeRef); + } + + auto flake = getFlake(state, flakeRef); + + if (done.count(flake.id)) continue; for (auto & require : flake.requires) todo.push(require); - done.emplace(flake.name, flake); + done.emplace(flake.id, flake); } return done; @@ -177,7 +192,7 @@ static std::map resolveFlakes(EvalState & state, const Strin static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - std::string flakeUri = state.forceStringNoCtx(*args[0], pos); + auto flakeUri = FlakeRef(state.forceStringNoCtx(*args[0], pos)); auto flakes = resolveFlakes(state, {flakeUri}); @@ -186,7 +201,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va state.mkAttrs(*vResult, flakes.size()); for (auto & flake : flakes) { - auto vFlake = state.allocAttr(*vResult, flake.second.name); + auto vFlake = state.allocAttr(*vResult, flake.second.id); state.mkAttrs(*vFlake, 2); mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh new file mode 100644 index 000000000..6be6e99d2 --- /dev/null +++ b/src/libexpr/primops/flake.hh @@ -0,0 +1,17 @@ +#include "types.hh" +#include "flakeref.hh" + +#include + +namespace nix { + +struct FlakeRegistry +{ + struct Entry + { + FlakeRef ref; + }; + std::map entries; +}; + +} diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc new file mode 100644 index 000000000..447b56822 --- /dev/null +++ b/src/libexpr/primops/flakeref.cc @@ -0,0 +1,139 @@ +#include "flakeref.hh" + +#include + +namespace nix { + +// A Git ref (i.e. branch or tag name). +const static std::string refRegex = "[a-zA-Z][a-zA-Z0-9_.-]*"; // FIXME: check + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +// A Git ref or revision. +const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; + +// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref +// optionally followed by a rev (e.g. "master" or +// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// GitHub references. +const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// URI stuff. +const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; +const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; +const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*"; + +FlakeRef::FlakeRef(const std::string & uri) +{ + // FIXME: could combine this into one regex. + + static std::regex flakeRegex( + "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?", + std::regex::ECMAScript); + + static std::regex githubRegex( + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", + std::regex::ECMAScript); + + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?://(" + authorityRegex + "))?" + + "(" + pathRegex + "))" + + "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?", + std::regex::ECMAScript); + + static std::regex refRegex2(refRegex, std::regex::ECMAScript); + + std::cmatch match; + if (std::regex_match(uri.c_str(), match, flakeRegex)) { + IsFlakeId d; + d.id = match[1]; + if (match[2].matched) + d.rev = Hash(match[2], htSHA1); + else if (match[3].matched) { + d.ref = match[3]; + if (match[4].matched) + d.rev = Hash(match[4], htSHA1); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, githubRegex)) { + IsGitHub d; + d.owner = match[1]; + d.repo = match[2]; + if (match[3].matched) + d.rev = Hash(match[3], htSHA1); + else if (match[4].matched) { + d.ref = match[4]; + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, uriRegex) && hasSuffix(match[4], ".git")) { + IsGit d; + d.uri = match[1]; + for (auto & param : tokenizeString(match[5], "&")) { + auto n = param.find('='); + assert(n != param.npos); + std::string name(param, 0, n); + std::string value(param, n + 1); + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw Error("invalid Git revision '%s'", value); + d.rev = Hash(value, htSHA1); + } else if (name == "ref") { + if (!std::regex_match(value, refRegex2)) + throw Error("invalid Git ref '%s'", value); + d.ref = value; + } else + // FIXME: should probably pass through unknown parameters + throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); + } + if (d.rev && !d.ref) + throw Error("flake URI '%s' lacks a Git ref", uri); + data = d; + } + + else + throw Error("'%s' is not a valid flake reference", uri); +} + +std::string FlakeRef::to_string() const +{ + if (auto refData = std::get_if(&data)) { + return + "flake:" + refData->id + + (refData->ref ? "/" + *refData->ref : "") + + (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); + } + + else if (auto refData = std::get_if(&data)) { + assert(!refData->ref || !refData->rev); + return + "github:" + refData->owner + "/" + refData->repo + + (refData->ref ? "/" + *refData->ref : "") + + (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); + } + + else if (auto refData = std::get_if(&data)) { + assert(refData->ref || !refData->rev); + return + refData->uri + + (refData->ref ? "?ref=" + *refData->ref : "") + + (refData->rev ? "&rev=" + refData->rev->to_string(Base16, false) : ""); + } + + else abort(); +} + +} diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh new file mode 100644 index 000000000..8559317e0 --- /dev/null +++ b/src/libexpr/primops/flakeref.hh @@ -0,0 +1,158 @@ +#include "types.hh" +#include "hash.hh" + +#include + +namespace nix { + +/* Flake references are a URI-like syntax to specify a flake. + + Examples: + + * (/rev-or-ref(/rev)?)? + + Look up a flake by ID in the flake lock file or in the flake + registry. These must specify an actual location for the flake + using the formats listed below. Note that in pure evaluation + mode, the flake registry is empty. + + Optionally, the rev or ref from the dereferenced flake can be + overriden. For example, + + nixpkgs/19.09 + + uses the "19.09" branch of the nixpkgs' flake GitHub repository, + while + + nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + uses the specified revision. For Git (rather than GitHub) + repositories, both the rev and ref must be given, e.g. + + nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + * github:/(/)? + + A repository on GitHub. These differ from Git references in that + they're downloaded in a efficient way (via the tarball mechanism) + and that they support downloading a specific revision without + specifying a branch. is either a commit hash ("rev") + or a branch or tag name ("ref"). The default is: "master" if none + is specified. Note that in pure evaluation mode, a commit hash + must be used. + + Flakes fetched in this manner expose "rev" and "lastModified" + attributes, but not "revCount". + + Examples: + + github:edolstra/dwarffs + github:edolstra/dwarffs/unstable + github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 + + * https:///.git(\?attr(&attr)*)? + ssh:///.git(\?attr(&attr)*)? + git:///.git(\?attr(&attr)*)? + file:///(\?attr(&attr)*)? + + where 'attr' is one of: + rev= + ref= + + A Git repository fetched through https. Note that the path must + end in ".git". The default for "ref" is "master". + + Examples: + + https://example.org/my/repo.git + https://example.org/my/repo.git?ref=release-1.2.3 + https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + + * /path.git(\?attr(&attr)*)? + + Like file://path.git, but if no "ref" or "rev" is specified, the + (possibly dirty) working tree will be used. Using a working tree + is not allowed in pure evaluation mode. + + Examples: + + /path/to/my/repo + /path/to/my/repo?ref=develop + /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + + * https:///.tar.xz(?hash=) + file:///.tar.xz(?hash=) + + A flake distributed as a tarball. In pure evaluation mode, an SRI + hash is mandatory. It exposes a "lastModified" attribute, being + the newest file inside the tarball. + + Example: + + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c + + Note: currently, there can be only one flake per Git repository, and + it must be at top-level. In the future, we may want to add a field + (e.g. "dir=") to specify a subdirectory inside the repository. +*/ + +typedef std::string FlakeId; + +struct FlakeRef +{ + struct IsFlakeId + { + FlakeId id; + std::optional ref; + std::optional rev; + }; + + struct IsGitHub + { + std::string owner, repo; + std::optional ref; + std::optional rev; + }; + + struct IsGit + { + std::string uri; + std::optional ref; + std::optional rev; + }; + + // Git, Tarball + + std::variant data; + + // Parse a flake URI. + FlakeRef(const std::string & uri); + + /* Unify two flake references so that the resulting reference + combines the information from both. For example, + "nixpkgs/" and "github:NixOS/nixpkgs" unifies to + "nixpkgs/master". May throw an exception if the references are + incompatible (e.g. "nixpkgs/" and "nixpkgs/", + where hash1 != hash2). */ + FlakeRef(const FlakeRef & a, const FlakeRef & b); + + // FIXME: change to operator <<. + std::string to_string() const; + + /* Check whether this is a "direct" flake reference, that is, not + a flake ID, which requires a lookup in the flake registry. */ + bool isDirect() const + { + return !std::get_if(&data); + } + + /* Check whether this is an "immutable" flake reference, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const + { + abort(); // TODO + } +}; + +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 98cd90c64..9b36c3cbd 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,3 +1,4 @@ +#include "primops/flake.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -27,7 +28,7 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs stopProgressBar(); for (auto & entry : registry.entries) { - std::cout << entry.first << " " << entry.second.uri << "\n"; + std::cout << entry.first << " " << entry.second.ref.to_string() << "\n"; } } }; From ba05f29838b3bafe28c3ea491be711229298cb1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 20:35:03 +0100 Subject: [PATCH 0026/2548] nix: Enable pure mode by default We want to encourage a brave new world of hermetic evaluation for source-level reproducibility, so flakes should not poke around in the filesystem outside of their explicit dependencies. Note that the default installation source remains impure in that it can refer to mutable flakes, so "nix build nixpkgs.hello" still works (and fetches the latest nixpkgs, unless it has been pinned by the user). A problem with pure evaluation is that builtins.currentSystem is unavailable. For the moment, I've hard-coded "x86_64-linux" in the nixpkgs flake. Eventually, "system" should be a flake function argument. --- corepkgs/default-installation-source.nix | 3 -- corepkgs/local.mk | 3 +- src/libexpr/eval.hh | 2 + src/libexpr/primops/flake.cc | 48 ++++++++++++------------ src/nix/installables.cc | 11 +++++- src/nix/main.cc | 1 + 6 files changed, 38 insertions(+), 30 deletions(-) delete mode 100644 corepkgs/default-installation-source.nix diff --git a/corepkgs/default-installation-source.nix b/corepkgs/default-installation-source.nix deleted file mode 100644 index 71ba04452..000000000 --- a/corepkgs/default-installation-source.nix +++ /dev/null @@ -1,3 +0,0 @@ -builtins.mapAttrs (flakeName: flakeInfo: - (getFlake flakeInfo.uri).${flakeName}.provides.packages or {}) - builtins.flakeRegistry diff --git a/corepkgs/local.mk b/corepkgs/local.mk index 41aaec63b..67306e50d 100644 --- a/corepkgs/local.mk +++ b/corepkgs/local.mk @@ -3,8 +3,7 @@ corepkgs_FILES = \ unpack-channel.nix \ derivation.nix \ fetchurl.nix \ - imported-drv-to-derivation.nix \ - default-installation-source.nix + imported-drv-to-derivation.nix $(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 35c01b97a..27c6c3da8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -318,6 +318,8 @@ public: const FlakeRegistry & getFlakeRegistry(); + Value * makeFlakeRegistryValue(); + private: std::unique_ptr _flakeRegistry; std::once_flag _flakeRegistryInit; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 5e92b1da3..4d027558d 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -16,50 +16,49 @@ const FlakeRegistry & EvalState::getFlakeRegistry() { _flakeRegistry = std::make_unique(); - if (!evalSettings.pureEval) { - #if 0 - auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; + auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; - auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); + auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); #endif - auto registryFile = readFile(settings.nixDataDir + "/nix/flake-registry.json"); + auto registryFile = readFile(settings.nixDataDir + "/nix/flake-registry.json"); - auto json = nlohmann::json::parse(registryFile); + auto json = nlohmann::json::parse(registryFile); - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", registryFile, version); + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", registryFile, version); - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - _flakeRegistry->entries.emplace(i.key(), entry); - } + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; + _flakeRegistry->entries.emplace(i.key(), entry); } }); return *_flakeRegistry; } -static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * args, Value & v) +Value * EvalState::makeFlakeRegistryValue() { - auto registry = state.getFlakeRegistry(); + auto v = allocValue(); - state.mkAttrs(v, registry.entries.size()); + auto registry = getFlakeRegistry(); + + mkAttrs(*v, registry.entries.size()); for (auto & entry : registry.entries) { - auto vEntry = state.allocAttr(v, entry.first); - state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); + auto vEntry = allocAttr(*v, entry.first); + mkAttrs(*vEntry, 2); + mkString(*allocAttr(*vEntry, symbols.create("uri")), entry.second.ref.to_string()); vEntry->attrs->sort(); } - v.attrs->sort(); -} + v->attrs->sort(); -static RegisterPrimOp r1("__flakeRegistry", 0, prim_flakeRegistry); + return v; +} static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef) { @@ -129,6 +128,9 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) auto flakePath = fetchFlake(state, flakeRef); state.store->assertStorePath(flakePath); + if (state.allowedPaths) + state.allowedPaths->insert(flakePath); + Flake flake; Value vInfo; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 9b7b96c25..faad057a7 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -30,8 +30,15 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) if (file != "") state.evalFile(lookupFileArg(state, file), *vSourceExpr); - else - state.evalFile(lookupFileArg(state, ""), *vSourceExpr); + else { + auto fun = state.parseExprFromString( + "builtins.mapAttrs (flakeName: flakeInfo:" + " (getFlake flakeInfo.uri).${flakeName}.provides.packages or {})", "/"); + auto vFun = state.allocValue(); + state.eval(fun, *vFun); + auto vRegistry = state.makeFlakeRegistryValue(); + mkApp(*vSourceExpr, *vFun, *vRegistry); + } return vSourceExpr; } diff --git a/src/nix/main.cc b/src/nix/main.cc index 4b909736d..01b0866f2 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -97,6 +97,7 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlError; settings.verboseBuild = false; + evalSettings.pureEval = true; NixArgs args; From 272b58220d17bc862f646dbc2cb38eea126001c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 21:05:44 +0100 Subject: [PATCH 0027/2548] Enforce use of immutable flakes in pure mode ... plus a temporary hack to allow impure flakes at top-level for the default installation source. --- src/libexpr/primops/fetchGit.cc | 6 +++--- src/libexpr/primops/fetchMercurial.cc | 6 +++--- src/libexpr/primops/flake.cc | 30 ++++++++++++++++++++------- src/libexpr/primops/flakeref.cc | 14 +++++++++++++ src/libexpr/primops/flakeref.hh | 5 +---- src/nix/installables.cc | 4 +++- 6 files changed, 46 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 62e9dfc0e..bbf13c87b 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,9 +22,6 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name) { - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { bool clean = true; @@ -218,6 +215,9 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + auto gitInfo = exportGit(state.store, url, ref, rev, name); state.mkAttrs(v, 8); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374..cfe1bd871 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -27,9 +27,6 @@ std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); HgInfo exportMercurial(ref store, const std::string & uri, std::string rev, const std::string & name) { - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; @@ -203,6 +200,9 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + auto hgInfo = exportMercurial(state.store, url, rev, name); state.mkAttrs(v, 8); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 4d027558d..1e70ccbd6 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -162,16 +162,17 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) return flake; } -/* Given a set of flake references, recursively fetch them and their +/* Given a flake reference, recursively fetch it and its dependencies. */ -static std::map resolveFlakes(EvalState & state, const std::vector & flakeRefs) +static std::map resolveFlake(EvalState & state, + const FlakeRef & topRef, bool impureTopRef) { std::map done; - std::queue todo; - for (auto & i : flakeRefs) todo.push(i); + std::queue> todo; + todo.push({topRef, impureTopRef}); while (!todo.empty()) { - auto flakeRef = todo.front(); + auto [flakeRef, impureRef] = todo.front(); todo.pop(); if (auto refData = std::get_if(&flakeRef.data)) { @@ -179,12 +180,15 @@ static std::map resolveFlakes(EvalState & state, const std::vect flakeRef = lookupFlake(state, flakeRef); } + if (evalSettings.pureEval && !flakeRef.isImmutable() && !impureRef) + throw Error("mutable flake '%s' is not allowed in pure mode; use --no-pure-eval to disable", flakeRef.to_string()); + auto flake = getFlake(state, flakeRef); if (done.count(flake.id)) continue; for (auto & require : flake.requires) - todo.push(require); + todo.push({require, false}); done.emplace(flake.id, flake); } @@ -194,9 +198,19 @@ static std::map resolveFlakes(EvalState & state, const std::vect static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - auto flakeUri = FlakeRef(state.forceStringNoCtx(*args[0], pos)); + auto flakeUri = state.forceStringNoCtx(*args[0], pos); - auto flakes = resolveFlakes(state, {flakeUri}); + // FIXME: temporary hack to make the default installation source + // work. + bool impure = false; + if (hasPrefix(flakeUri, "impure:")) { + flakeUri = std::string(flakeUri, 7); + impure = true; + } + + auto flakeRef = FlakeRef(flakeUri); + + auto flakes = resolveFlake(state, flakeUri, impure); auto vResult = state.allocValue(); diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 447b56822..639313f21 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -136,4 +136,18 @@ std::string FlakeRef::to_string() const else abort(); } +bool FlakeRef::isImmutable() const +{ + if (auto refData = std::get_if(&data)) + return (bool) refData->rev; + + else if (auto refData = std::get_if(&data)) + return (bool) refData->rev; + + else if (auto refData = std::get_if(&data)) + return (bool) refData->rev; + + else abort(); +} + } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 8559317e0..ad0cf8630 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -149,10 +149,7 @@ struct FlakeRef /* Check whether this is an "immutable" flake reference, that is, one that contains a commit hash or content hash. */ - bool isImmutable() const - { - abort(); // TODO - } + bool isImmutable() const; }; } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index faad057a7..b4584f168 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -31,9 +31,11 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) if (file != "") state.evalFile(lookupFileArg(state, file), *vSourceExpr); else { + // FIXME: remove "impure" hack, call some non-user-accessible + // variant of getFlake instead. auto fun = state.parseExprFromString( "builtins.mapAttrs (flakeName: flakeInfo:" - " (getFlake flakeInfo.uri).${flakeName}.provides.packages or {})", "/"); + " (getFlake (\"impure:\" + flakeInfo.uri)).${flakeName}.provides.packages or {})", "/"); auto vFun = state.allocValue(); state.eval(fun, *vFun); auto vRegistry = state.makeFlakeRegistryValue(); From beab05851bfa895fe538f15f8bbb2da3a20db638 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 21:55:43 +0100 Subject: [PATCH 0028/2548] nix: Add --flake flag This allows using an arbitrary "provides" attribute from the specified flake. For example: nix build --flake nixpkgs packages.hello (Maybe provides.packages should be used for consistency...) --- src/libexpr/eval.hh | 2 -- src/libexpr/primops/flake.cc | 52 ++++++++++++++++++++++++------------ src/libexpr/primops/flake.hh | 7 +++++ src/nix/command.hh | 3 ++- src/nix/installables.cc | 26 ++++++++++++++---- 5 files changed, 65 insertions(+), 25 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 27c6c3da8..35c01b97a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -318,8 +318,6 @@ public: const FlakeRegistry & getFlakeRegistry(); - Value * makeFlakeRegistryValue(); - private: std::unique_ptr _flakeRegistry; std::once_flag _flakeRegistryInit; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1e70ccbd6..3bd62a50b 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -40,18 +40,18 @@ const FlakeRegistry & EvalState::getFlakeRegistry() return *_flakeRegistry; } -Value * EvalState::makeFlakeRegistryValue() +Value * makeFlakeRegistryValue(EvalState & state) { - auto v = allocValue(); + auto v = state.allocValue(); - auto registry = getFlakeRegistry(); + auto registry = state.getFlakeRegistry(); - mkAttrs(*v, registry.entries.size()); + state.mkAttrs(*v, registry.entries.size()); for (auto & entry : registry.entries) { - auto vEntry = allocAttr(*v, entry.first); - mkAttrs(*vEntry, 2); - mkString(*allocAttr(*vEntry, symbols.create("uri")), entry.second.ref.to_string()); + auto vEntry = state.allocAttr(*v, entry.first); + state.mkAttrs(*vEntry, 2); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); vEntry->attrs->sort(); } @@ -163,16 +163,19 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } /* Given a flake reference, recursively fetch it and its - dependencies. */ -static std::map resolveFlake(EvalState & state, + dependencies. + FIXME: this should return a graph of flakes. +*/ +static std::tuple> resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) { std::map done; std::queue> todo; - todo.push({topRef, impureTopRef}); + std::optional topFlakeId; /// FIXME: ambiguous + todo.push({topRef, true}); while (!todo.empty()) { - auto [flakeRef, impureRef] = todo.front(); + auto [flakeRef, toplevel] = todo.front(); todo.pop(); if (auto refData = std::get_if(&flakeRef.data)) { @@ -180,26 +183,27 @@ static std::map resolveFlake(EvalState & state, flakeRef = lookupFlake(state, flakeRef); } - if (evalSettings.pureEval && !flakeRef.isImmutable() && !impureRef) + if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --no-pure-eval to disable", flakeRef.to_string()); auto flake = getFlake(state, flakeRef); if (done.count(flake.id)) continue; + if (toplevel) topFlakeId = flake.id; + for (auto & require : flake.requires) todo.push({require, false}); done.emplace(flake.id, flake); } - return done; + assert(topFlakeId); + return {*topFlakeId, done}; } -static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v) { - auto flakeUri = state.forceStringNoCtx(*args[0], pos); - // FIXME: temporary hack to make the default installation source // work. bool impure = false; @@ -210,14 +214,20 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va auto flakeRef = FlakeRef(flakeUri); - auto flakes = resolveFlake(state, flakeUri, impure); + auto [topFlakeId, flakes] = resolveFlake(state, flakeUri, impure); + + // FIXME: we should call each flake with only its dependencies + // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); state.mkAttrs(*vResult, flakes.size()); + Value * vTop = 0; + for (auto & flake : flakes) { auto vFlake = state.allocAttr(*vResult, flake.second.id); + if (topFlakeId == flake.second.id) vTop = vFlake; state.mkAttrs(*vFlake, 2); mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); @@ -228,6 +238,14 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va vResult->attrs->sort(); v = *vResult; + + assert(vTop); + return vTop; +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 6be6e99d2..e504dc196 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -5,6 +5,9 @@ namespace nix { +struct Value; +class EvalState; + struct FlakeRegistry { struct Entry @@ -14,4 +17,8 @@ struct FlakeRegistry std::map entries; }; +Value * makeFlakeRegistryValue(EvalState & state); + +Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); + } diff --git a/src/nix/command.hh b/src/nix/command.hh index 04183c7ed..a08347945 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -53,7 +53,8 @@ struct Installable struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { - Path file; + std::optional file; + std::optional flakeUri; SourceExprCommand(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b4584f168..0453c72c2 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "primops/flake.hh" #include @@ -18,8 +19,15 @@ SourceExprCommand::SourceExprCommand() .shortName('f') .longName("file") .label("file") - .description("evaluate FILE rather than the default") + .description("evaluate FILE rather than use the default installation source") .dest(&file); + + mkFlag() + .shortName('F') + .longName("flake") + .label("flake") + .description("evaluate FLAKE rather than use the default installation source") + .dest(&flakeUri); } Value * SourceExprCommand::getSourceExpr(EvalState & state) @@ -28,9 +36,17 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) vSourceExpr = state.allocValue(); - if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); - else { + if (file && flakeUri) + throw Error("cannot use both --file and --flake"); + + if (file) + state.evalFile(lookupFileArg(state, *file), *vSourceExpr); + else if (flakeUri) { + // FIXME: handle flakeUri being a relative path + auto vTemp = state.allocValue(); + auto vFlake = *makeFlakeValue(state, "impure:" + *flakeUri, *vTemp); + *vSourceExpr = *((*vFlake.attrs->get(state.symbols.create("provides")))->value); + } else { // FIXME: remove "impure" hack, call some non-user-accessible // variant of getFlake instead. auto fun = state.parseExprFromString( @@ -38,7 +54,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) " (getFlake (\"impure:\" + flakeInfo.uri)).${flakeName}.provides.packages or {})", "/"); auto vFun = state.allocValue(); state.eval(fun, *vFun); - auto vRegistry = state.makeFlakeRegistryValue(); + auto vRegistry = makeFlakeRegistryValue(state); mkApp(*vSourceExpr, *vFun, *vRegistry); } From e38ec77de8077658d2f75fd6ff8b0f0f06babda9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 22:06:19 +0100 Subject: [PATCH 0029/2548] Interpret all file:// URIs as Git repositories --- src/libexpr/primops/flakeref.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 639313f21..a2700f102 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -79,7 +79,9 @@ FlakeRef::FlakeRef(const std::string & uri) data = d; } - else if (std::regex_match(uri.c_str(), match, uriRegex) && hasSuffix(match[4], ".git")) { + else if (std::regex_match(uri.c_str(), match, uriRegex) + && (match[2] == "file" || hasSuffix(match[4], ".git"))) + { IsGit d; d.uri = match[1]; for (auto & param : tokenizeString(match[5], "&")) { From 6e9182fbc2544e5252366321da1c9406571d01e4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 22:43:22 +0100 Subject: [PATCH 0030/2548] Add basic flake lock file support --- src/libexpr/primops/flake.cc | 117 +++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 41 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3bd62a50b..a788e935e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -10,31 +10,40 @@ namespace nix { +/* Read the registry or a lock file. (Currently they have an identical + format. */ +static std::unique_ptr readRegistry(const Path & path) +{ + auto registry = std::make_unique(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; + registry->entries.emplace(i.key(), entry); + } + + return registry; +} + const FlakeRegistry & EvalState::getFlakeRegistry() { std::call_once(_flakeRegistryInit, [&]() { - _flakeRegistry = std::make_unique(); - #if 0 auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); #endif - auto registryFile = readFile(settings.nixDataDir + "/nix/flake-registry.json"); + auto registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - auto json = nlohmann::json::parse(registryFile); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", registryFile, version); - - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - _flakeRegistry->entries.emplace(i.key(), entry); - } + _flakeRegistry = readRegistry(registryFile); }); return *_flakeRegistry; @@ -60,33 +69,24 @@ Value * makeFlakeRegistryValue(EvalState & state) return v; } -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef) +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, + std::vector registries) { if (auto refData = std::get_if(&flakeRef.data)) { - auto registry = state.getFlakeRegistry(); - auto i = registry.entries.find(refData->id); - if (i == registry.entries.end()) - throw Error("cannot find flake '%s' in the flake registry", refData->id); - auto newRef = FlakeRef(i->second.ref); - if (!newRef.isDirect()) - throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); - return newRef; + for (auto registry : registries) { + auto i = registry->entries.find(refData->id); + if (i != registry->entries.end()) { + auto newRef = FlakeRef(i->second.ref); + if (!newRef.isDirect()) + throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); + return newRef; + } + } + throw Error("cannot find flake '%s' in the flake registry or in the flake lock file", refData->id); } else return flakeRef; } -struct Flake -{ - FlakeId id; - std::string description; - Path path; - std::set requires; - Value * vProvides; // FIXME: gc - // commit hash - // date - // content hash -}; - static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) { assert(flakeRef.isDirect()); @@ -123,6 +123,19 @@ static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } +struct Flake +{ + FlakeId id; + std::string description; + Path path; + std::vector requires; + std::unique_ptr lockFile; + Value * vProvides; // FIXME: gc + // commit hash + // date + // content hash +}; + static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { auto flakePath = fetchFlake(state, flakeRef); @@ -134,7 +147,7 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) Flake flake; Value vInfo; - state.evalFile(flakePath + "/flake.nix", vInfo); + state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack state.forceAttrs(vInfo); @@ -149,8 +162,8 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) { state.forceList(*(**requires).value, *(**requires).pos); for (unsigned int n = 0; n < (**requires).value->listSize(); ++n) - flake.requires.insert(state.forceStringNoCtx( - *(**requires).value->listElems()[n], *(**requires).pos)); + flake.requires.push_back(FlakeRef(state.forceStringNoCtx( + *(**requires).value->listElems()[n], *(**requires).pos))); } if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { @@ -159,6 +172,16 @@ static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake lacks attribute 'provides'"); + auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack + + if (pathExists(lockFile)) { + flake.lockFile = readRegistry(lockFile); + for (auto & entry : flake.lockFile->entries) + if (!entry.second.ref.isImmutable()) + throw Error("flake lock file '%s' contains mutable entry '%s'", + lockFile, entry.second.ref.to_string()); + } + return flake; } @@ -174,13 +197,19 @@ static std::tuple> resolveFlake(EvalState & st std::optional topFlakeId; /// FIXME: ambiguous todo.push({topRef, true}); + std::vector registries; + FlakeRegistry localRegistry; + registries.push_back(&localRegistry); + if (!evalSettings.pureEval) + registries.push_back(&state.getFlakeRegistry()); + while (!todo.empty()) { auto [flakeRef, toplevel] = todo.front(); todo.pop(); if (auto refData = std::get_if(&flakeRef.data)) { if (done.count(refData->id)) continue; // optimization - flakeRef = lookupFlake(state, flakeRef); + flakeRef = lookupFlake(state, flakeRef, registries); } if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) @@ -195,11 +224,17 @@ static std::tuple> resolveFlake(EvalState & st for (auto & require : flake.requires) todo.push({require, false}); - done.emplace(flake.id, flake); + if (flake.lockFile) + for (auto & entry : flake.lockFile->entries) { + if (localRegistry.entries.count(entry.first)) continue; + localRegistry.entries.emplace(entry.first, entry.second); + } + + done.emplace(flake.id, std::move(flake)); } assert(topFlakeId); - return {*topFlakeId, done}; + return {*topFlakeId, std::move(done)}; } Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v) From 7ce1fae59f6cb4341f61a48a5216cb252e4eca54 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 13 Feb 2019 00:03:10 -0500 Subject: [PATCH 0031/2548] Support --disable-shared flag. This tells Nix to not build the shared libraries. --- Makefile.config.in | 2 ++ configure.ac | 10 ++++++++++ mk/libraries.mk | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index b01a4afbf..59730b646 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,4 +1,6 @@ +AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ diff --git a/configure.ac b/configure.ac index 5a2526672..410b20972 100644 --- a/configure.ac +++ b/configure.ac @@ -64,6 +64,7 @@ AC_PROG_CXX AC_PROG_CPP AX_CXX_COMPILE_STDCXX_14 +AC_CHECK_TOOL([AR], [ar]) # Use 64-bit file system calls so that we can support files > 2 GiB. AC_SYS_LARGEFILE @@ -267,6 +268,15 @@ AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], sandbox_shell=$withval) AC_SUBST(sandbox_shell) +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) +fi + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix diff --git a/mk/libraries.mk b/mk/libraries.mk index 14c95fa91..3953446cb 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) ar crs $$@ $$? + $(trace-ar) $(AR) crs $$@ $$? $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) From e58a71442ad4a538b48fc7a9938c3690628c4741 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 14 Feb 2019 13:24:16 -0500 Subject: [PATCH 0032/2548] nix.sh: Be set -u compliant. --- scripts/nix-profile.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index a5f52274f..db03e16ba 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -75,7 +75,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi - if [ -n "${MANPATH}" ]; then + if [ -n "${MANPATH-}" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi From b9567aa8b6f935522d1037377e0e5205a7529fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 13 Feb 2019 11:28:28 +0100 Subject: [PATCH 0033/2548] install script: don't abort when "nix-channel --update" fails Instead, print a message about what happened and tell the user what can be done (run "[sudo -i] nix-channel --update nixpkgs" again at a later time). This change allows installing Nix when you're offline. Since the multi-user installer is so verbose, the message isn't printed until the end. Fixes issue #2650 ("installation without internet connection"). --- scripts/install-multi-user.sh | 10 +++++++++- scripts/install-nix-from-closure.sh | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..3d2acaac8 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -240,10 +240,16 @@ EOF } trap finish_fail EXIT +channel_update_failed=0 function finish_success { finish_cleanup ok "Alright! We're done!" + if [ "x$channel_update_failed" = x1 ]; then + echo "" + echo "But fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." + fi cat < Date: Fri, 22 Feb 2019 21:07:48 +0100 Subject: [PATCH 0034/2548] remove noop uses of nix-store --init the nix-store --init command is a noop apparently --- release.nix | 1 - scripts/install-multi-user.sh | 3 --- scripts/install-nix-from-closure.sh | 6 ------ tests/common.sh.in | 1 - tests/init.sh | 3 --- 5 files changed, 14 deletions(-) diff --git a/release.nix b/release.nix index 271645067..5c0027301 100644 --- a/release.nix +++ b/release.nix @@ -278,7 +278,6 @@ let pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; } '' export NIX_STATE_DIR=$TMPDIR - nix-store --init nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \ --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }' diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..f93be9d3a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -674,9 +674,6 @@ $NIX_INSTALLED_NIX. EOF fi - _sudo "to initialize the Nix Database" \ - $NIX_INSTALLED_NIX/bin/nix-store --init - cat ./.reginfo \ | _sudo "to load data for the first time in to the Nix Database" \ "$NIX_INSTALLED_NIX/bin/nix-store" --load-db diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index ab20774bb..fc633fa23 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -109,12 +109,6 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do done echo "" >&2 -echo "initialising Nix database..." >&2 -if ! $nix/bin/nix-store --init; then - echo "$0: failed to initialize the Nix database" >&2 - exit 1 -fi - if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then echo "$0: unable to register valid paths" >&2 exit 1 diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..28d0e8bfd 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -53,7 +53,6 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" - nix-store --init clearProfiles } diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..6fa4f815b 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -25,9 +25,6 @@ fsync-metadata = false !include nix.conf.extra.not-there EOF -# Initialise the database. -nix-store --init - # Did anything happen? test -e "$NIX_STATE_DIR"/db/db.sqlite From d7a7a029ff9c1fc6c5a22b91718e28c4c1062d9f Mon Sep 17 00:00:00 2001 From: Austin Seipp Date: Fri, 22 Feb 2019 13:07:35 -0600 Subject: [PATCH 0035/2548] scripts: remove default 'sandbox = false' from multi-user installer Sandboxing is now enabled by default on Linux, but is still disabled on macOS. However, the installer always turned it off to ensure consistent behavior. Remove this default configuration, so we fall back to the default platform-specific value. Signed-off-by: Austin Seipp --- scripts/install-multi-user.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6ee8dd485..4c793247e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -747,7 +747,6 @@ build-users-group = $NIX_BUILD_GROUP_NAME max-jobs = $NIX_USER_COUNT cores = 1 -sandbox = false EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf From 529add316c5356a8060c35f987643b7bf5c796dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:20:50 +0800 Subject: [PATCH 0036/2548] downloadCached: Return ETag This allows fetchFlake() to return the Git revision of a GitHub archive. --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/flake.cc | 49 +++++++++++++++++++++++---------- src/libstore/download.cc | 17 +++++++++--- src/libstore/download.hh | 12 ++++++-- src/nix-channel/nix-channel.cc | 6 ++-- 7 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..37c74a94b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); + return getDownloader()->downloadCached(state.store, s, true).path; else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index cbd576d7d..0f1ac05d6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -657,7 +657,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f787ad96b..7e3dd23ed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2112,7 +2112,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index a788e935e..f9a1a3d6a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -87,7 +87,13 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return flakeRef; } -static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) +struct FlakeSourceInfo +{ + Path storePath; + std::optional rev; +}; + +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) { assert(flakeRef.isDirect()); @@ -99,25 +105,36 @@ static Path fetchFlake(EvalState & state, const FlakeRef & flakeRef) // FIXME: support passing auth tokens for private repos. - auto storePath = getDownloader()->downloadCached(state.store, - fmt("https://api.github.com/repos/%s/%s/tarball/%s", - refData->owner, refData->repo, - refData->rev - ? refData->rev->to_string(Base16, false) - : refData->ref - ? *refData->ref - : "master"), - true, "source"); + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + refData->rev + ? refData->rev->to_string(Base16, false) + : refData->ref + ? *refData->ref + : "master"); - // FIXME: extract revision hash from ETag. + auto result = getDownloader()->downloadCached(state.store, url, true, "source"); - return storePath; + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeSourceInfo info; + info.storePath = result.path; + info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + + return info; } else if (auto refData = std::get_if(&flakeRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, refData->ref, refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); - return gitInfo.storePath; + FlakeSourceInfo info; + info.storePath = gitInfo.storePath; + info.rev = Hash(gitInfo.rev, htSHA1); + return info; } else abort(); @@ -138,7 +155,11 @@ struct Flake static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - auto flakePath = fetchFlake(state, flakeRef); + auto sourceInfo = fetchFlake(state, flakeRef); + debug("got flake source '%s' with revision %s", + sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + + auto flakePath = sourceInfo.storePath; state.store->assertStorePath(flakePath); if (state.allowedPaths) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..360d48b09 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -771,7 +771,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -783,8 +783,11 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Path expectedStorePath; if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -803,6 +806,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -814,6 +819,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa skip = true; if (effectiveUrl) *effectiveUrl = url_; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -831,6 +837,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa auto res = download(request); if (effectiveUrl) *effectiveUrl = res.effectiveUrl; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; @@ -852,6 +859,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } @@ -885,7 +893,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa url, expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..8acfe4e1a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -41,6 +41,12 @@ struct DownloadResult uint64_t bodySize = 0; }; +struct CachedDownloadResult +{ + Path path; + std::optional etag; +}; + class Store; struct Downloader @@ -64,8 +70,10 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached( + ref store, const string & uri, bool unpack, string name = "", + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, + int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..7b23088a2 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -88,7 +88,7 @@ static void update(const StringSet & channelNames) // definition from a consistent location if the redirect changes mid-download. std::string effectiveUrl; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); + auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; url = chomp(std::move(effectiveUrl)); // If the URL contains a version number, append it to the name @@ -123,10 +123,10 @@ static void update(const StringSet & channelNames) // Download the channel tarball. auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } catch (DownloadError & e) { fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } chomp(filename); } From d342de02b9f7ee07f22e1986af8d5c8eb325d8ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:23:45 +0800 Subject: [PATCH 0037/2548] fetchFlake: Use infinite TTL when the revision is specified --- src/libexpr/primops/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f9a1a3d6a..3998f9ef9 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -113,7 +113,8 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) ? *refData->ref : "master"); - auto result = getDownloader()->downloadCached(state.store, url, true, "source"); + auto result = getDownloader()->downloadCached(state.store, url, true, "source", + Hash(), nullptr, refData->rev ? 1000000000 : settings.tarballTtl); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); From 07f992a74b64f4376d5b415d0042babc924772f3 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 11:44:25 +0100 Subject: [PATCH 0038/2548] Extract and expose splitUriAndParams function which splits a URL into localtor and parameter parts --- src/libstore/store-api.cc | 16 ++++++++++++---- src/libstore/store-api.hh | 4 ++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 913a11121..c13ff1156 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -842,12 +842,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString(uri.substr(q + 1), "&")) { @@ -873,6 +872,15 @@ ref openStore(const std::string & uri_, } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f504735e0..ad0f8df11 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -798,4 +798,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri); + } From 56c18c67d98078dbed1d05ac68663cc52d2cb543 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Mon, 25 Feb 2019 17:56:52 +0100 Subject: [PATCH 0039/2548] Extend S3 URL parsing with parameters extraction --- src/libstore/download.cc | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 467f570bb..8bc496515 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -614,6 +614,22 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, Callback callback) override { @@ -622,12 +638,8 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From ac200c3678c0f569cd962d8bbd22bb74b223d748 Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Thu, 21 Feb 2019 14:26:50 +0100 Subject: [PATCH 0040/2548] Apply param values on S3Helper initialization --- src/libstore/download.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8bc496515..f1666d293 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -639,7 +639,14 @@ struct CurlDownloader : public Downloader try { #ifdef ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - S3Helper s3Helper("", Aws::Region::US_EAST_1, "", ""); // FIXME: make configurable + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; From 06d633598727763c54b4b049dbc213106474d10c Mon Sep 17 00:00:00 2001 From: Dzmitry Zaitsau Date: Tue, 26 Feb 2019 11:04:18 +0100 Subject: [PATCH 0041/2548] fix indentation --- src/libstore/download.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index f1666d293..22382ab1d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -620,8 +620,8 @@ struct CurlDownloader : public Downloader auto [path, params] = splitUriAndParams(uri); auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", path); + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); std::string bucketName(path, 5, slash - 5); std::string key(path, slash + 1); From cfb6ab80cea7f0ed3f525e8120f2e569f963fa0e Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Feb 2019 06:53:01 +0100 Subject: [PATCH 0042/2548] Implemented "nix flake info" --- src/libexpr/primops/flake.cc | 13 ------------- src/libexpr/primops/flake.hh | 14 ++++++++++++++ src/nix/command.hh | 11 +++++++++++ src/nix/flake.cc | 23 ++++++++++++++++++++++- 4 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3998f9ef9..9f137a0b9 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -141,19 +141,6 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } -struct Flake -{ - FlakeId id; - std::string description; - Path path; - std::vector requires; - std::unique_ptr lockFile; - Value * vProvides; // FIXME: gc - // commit hash - // date - // content hash -}; - static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { auto sourceInfo = fetchFlake(state, flakeRef); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index e504dc196..af28bc5b0 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -21,4 +21,18 @@ Value * makeFlakeRegistryValue(EvalState & state); Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); +struct Flake +{ + FlakeId id; + std::string description; + Path path; + std::vector requires; + std::unique_ptr lockFile; + Value * vProvides; // FIXME: gc + // commit hash + // date + // content hash +}; + +static Flake getFlake(EvalState & state, const FlakeRef & flakeRef); } diff --git a/src/nix/command.hh b/src/nix/command.hh index a08347945..b3248222e 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -34,6 +34,17 @@ struct Buildable typedef std::vector Buildables; +struct FlakeCommand : virtual Args, StoreCommand, MixEvalArgs +{ + std::string flakeUri; + +public: + FlakeCommand() + { + expectArg("flake-uri", &flakeUri); + } +}; + struct Installable { virtual std::string what() = 0; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9b36c3cbd..099425688 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -33,10 +33,31 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; +struct CmdFlakeInfo : FlakeCommand +{ + std::string name() override + { + return "info"; + } + + std::string description() override + { + return "list info about a given flake"; + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + std::cout << "Location: " << flake.path << "\n"; + std::cout << "Description: " << flake.description << "\n"; + } +}; + struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() - : MultiCommand({make_ref()}) + : MultiCommand({make_ref(), make_ref()}) { } From 0963479741d476393260b831a242c74c918ed6e6 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 4 Apr 2018 13:48:50 -0500 Subject: [PATCH 0043/2548] archive.cc: ignore more posix_fallocate "not supported" error codes Fixes w/musl. --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index bb68e8288..3aa120270 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -331,7 +331,7 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) throw SysError(format("preallocating file of %1% bytes") % len); } #endif From 4cfc131ec43ccbfa2f87efc3e96bdbe127ab3b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Fri, 1 Mar 2019 14:30:30 +0100 Subject: [PATCH 0044/2548] manual nitpick: document sha512 support in hashString --- doc/manual/expressions/builtins.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 412622714..22d998bd0 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -693,8 +693,8 @@ builtins.genList (x: x * x) 5 Return a base-16 representation of the cryptographic hash of string s. The hash algorithm specified by type must - be one of "md5", "sha1" or - "sha256". + be one of "md5", "sha1", + "sha256" or "sha512". From b71e1fb342b2344b738733698b20c5ec03dabe5a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2019 18:27:45 +0800 Subject: [PATCH 0045/2548] Restore --init calls in tests --- tests/common.sh.in | 1 + tests/init.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/common.sh.in b/tests/common.sh.in index 28d0e8bfd..6a523ca9d 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -53,6 +53,7 @@ clearStore() { mkdir "$NIX_STORE_DIR" rm -rf "$NIX_STATE_DIR" mkdir "$NIX_STATE_DIR" + nix-store --init clearProfiles } diff --git a/tests/init.sh b/tests/init.sh index 6fa4f815b..19a12c1e2 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -25,6 +25,9 @@ fsync-metadata = false !include nix.conf.extra.not-there EOF +# Initialise the database. +nix-store --init + # Did anything happen? test -e "$NIX_STATE_DIR"/db/db.sqlite From fff8db205c393c53e42326f274b322eb70259250 Mon Sep 17 00:00:00 2001 From: volth Date: Wed, 6 Mar 2019 05:03:25 +0000 Subject: [PATCH 0046/2548] canBuildLocally: check for features It could happen that the local builder match the system but lacks some features. Now it results a failure. The fix gracefully excludes the local builder from the set of available builders for derivation which requires the feature, so the derivation is built on remote builders only (as though it has incompatible system, like ```aarch64-linux``` when local is x86) --- src/build-remote/build-remote.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index abf3669b5..279ae62f6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -38,6 +38,12 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } +static bool allSupportedLocally(const std::set& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} + static int _main(int argc, char * * argv) { { @@ -97,9 +103,10 @@ static int _main(int argc, char * * argv) source >> drvPath; auto requiredFeatures = readStrings>(source); - auto canBuildLocally = amWilling - && ( neededSystem == settings.thisSystem - || settings.extraPlatforms.get().count(neededSystem) > 0); + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); From 9ff1a9ea65bdeb520becb843b8300a23fb88a5a9 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 27 Feb 2019 19:54:18 +0100 Subject: [PATCH 0047/2548] Implemented json flag for `nix flake info` --- src/libexpr/primops/flake.cc | 2 +- src/libexpr/primops/flake.hh | 2 +- src/nix/command.cc | 9 +++++++++ src/nix/command.hh | 7 +++++++ src/nix/flake.cc | 14 +++++++++++--- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 9f137a0b9..ac0421549 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -141,7 +141,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } -static Flake getFlake(EvalState & state, const FlakeRef & flakeRef) +Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { auto sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index af28bc5b0..194b969a2 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -34,5 +34,5 @@ struct Flake // content hash }; -static Flake getFlake(EvalState & state, const FlakeRef & flakeRef); +Flake getFlake(EvalState & state, const FlakeRef & flakeRef); } diff --git a/src/nix/command.cc b/src/nix/command.cc index 5967ab36c..e1e32aaae 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -27,6 +27,15 @@ void StoreCommand::run() run(getStore()); } +JsonFormattable::JsonFormattable() +{ + mkFlag() + .longName("json-formattable") + .shortName('j') + .description("output will be printed as json") + .handler([&]() { jsonFormatting = true; }); +} + StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { diff --git a/src/nix/command.hh b/src/nix/command.hh index b3248222e..5c2f8c304 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -26,6 +26,13 @@ private: std::shared_ptr _store; }; +struct JsonFormattable : virtual Command +{ + bool jsonFormatting = false;; + + JsonFormattable(); +}; + struct Buildable { Path drvPath; // may be empty diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 099425688..22e5b297c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "progress-bar.hh" #include "eval.hh" +#include using namespace nix; @@ -33,7 +34,7 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; -struct CmdFlakeInfo : FlakeCommand +struct CmdFlakeInfo : FlakeCommand, JsonFormattable { std::string name() override { @@ -49,8 +50,15 @@ struct CmdFlakeInfo : FlakeCommand { auto evalState = std::make_shared(searchPath, store); nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); - std::cout << "Location: " << flake.path << "\n"; - std::cout << "Description: " << flake.description << "\n"; + if (jsonFormatting) { + nlohmann::json j; + j["location"] = flake.path; + j["description"] = flake.description; + std::cout << j.dump(4) << std::endl; + } else { + std::cout << "Location: " << flake.path << "\n"; + std::cout << "Description: " << flake.description << "\n"; + } } }; From 6542de98c298b6dc268b358166bd2f5bea2cc230 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Mon, 25 Feb 2019 13:46:37 +0100 Subject: [PATCH 0048/2548] Implemented writeRegistry --- src/libexpr/primops/flake.cc | 12 ++++++++++++ src/libexpr/primops/flake.hh | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index ac0421549..9d1da84f1 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -31,6 +31,18 @@ static std::unique_ptr readRegistry(const Path & path) return registry; } +/* Write the registry or lock file to a file. */ +static void writeRegistry(FlakeRegistry registry, Path path = "./flake.lock") +{ + nlohmann::json json = {}; + json["value"] = 0; // Not sure whether this should be 0. + json["flakes"] = {}; + for (auto elem : registry.entries) { + json["flakes"][elem.first] = elem.second.ref.to_string(); + } + writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. +} + const FlakeRegistry & EvalState::getFlakeRegistry() { std::call_once(_flakeRegistryInit, [&]() diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 194b969a2..90c6bc7d2 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -35,4 +35,6 @@ struct Flake }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef); + +void writeRegistry(FlakeRegistry); } From 43331d634498154f6a1835c8c417dd07eaefd45d Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Wed, 27 Feb 2019 23:32:12 +0100 Subject: [PATCH 0049/2548] detail memory roots --- src/libstore/gc.cc | 61 +++++++++++++++++++------------------ src/libstore/local-store.hh | 2 +- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b415d5421..ecfa5e1ed 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -333,9 +333,7 @@ Roots LocalStore::findRootsNoTemp() NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ - size_t n = 0; - for (auto & root : findRuntimeRoots()) - roots[fmt("{memory:%d}", n++)] = root; + findRuntimeRoots(roots); return roots; } @@ -357,8 +355,7 @@ Roots LocalStore::findRoots() return roots; } - -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, Roots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -377,7 +374,7 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast(buf), res); + roots.emplace(file, std::string(static_cast(buf), res)); return; } @@ -387,20 +384,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, Roots & roots) { try { - paths.emplace(readFile(path)); + roots.emplace(path, readFile(path)); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -PathSet LocalStore::findRuntimeRoots() +void LocalStore::findRuntimeRoots(Roots & roots) { - PathSet roots; - StringSet paths; + Roots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -410,8 +407,8 @@ PathSet LocalStore::findRuntimeRoots() while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked); + readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked); auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); @@ -423,7 +420,7 @@ PathSet LocalStore::findRuntimeRoots() struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); + readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked); } } if (errno) { @@ -434,18 +431,22 @@ PathSet LocalStore::findRuntimeRoots() fdDir.reset(); try { - auto mapLines = - tokenizeString>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); + auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); + auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); + int n = 0; for (const auto& line : mapLines) { + n++; auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); + unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]); } - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); + auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; + n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -462,10 +463,11 @@ PathSet LocalStore::findRuntimeRoots() std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + int n = 0; for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); + unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -473,21 +475,20 @@ PathSet LocalStore::findRuntimeRoots() #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { + for (auto & root : unchecked) { + if (isInStore(root.second)) { + Path path = toStorePath(root.second); + if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.insert(path); + roots.emplace(root.first, path); } } - - return roots; + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fce963433..730d4c917 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -269,7 +269,7 @@ private: Roots findRootsNoTemp(); - PathSet findRuntimeRoots(); + void findRuntimeRoots(Roots & roots); void removeUnusedLinks(const GCState & state); From a17f86ce3a67dd2dab2329d7262bc4ad4e7c37ff Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 28 Feb 2019 23:26:07 +0100 Subject: [PATCH 0050/2548] Obfuscate memory roots for non-root users --- src/libstore/gc.cc | 14 +++++++------- src/nix-daemon/nix-daemon.cc | 11 +++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ecfa5e1ed..73630f36d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -374,7 +374,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - roots.emplace(file, std::string(static_cast(buf), res)); + roots.emplace((format("{memory:%1%") % file).str(), + std::string(static_cast(buf), res)); return; } @@ -407,8 +408,8 @@ void LocalStore::findRuntimeRoots(Roots & roots) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), unchecked); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), unchecked); + readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked); + readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked); auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); @@ -435,10 +436,9 @@ void LocalStore::findRuntimeRoots(Roots & roots) auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); int n = 0; for (const auto& line : mapLines) { - n++; auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked.emplace((format("{%1%:%2%}") % mapFile % n).str(), match[1]); + unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]); } auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); @@ -446,7 +446,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) auto env_end = std::sregex_iterator{}; n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked.emplace((format("{%1%:%2%}") % envFile % envString).str(), i->str()); + unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -467,7 +467,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked.emplace((format("{%1%:%2%}" % LSOF % n++).str(), match[1]); + unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8368c3266..faa23b268 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -478,8 +478,15 @@ static void performOp(TunnelLogger * logger, ref store, Roots roots = store->findRoots(); logger->stopWork(); to << roots.size(); - for (auto & i : roots) - to << i.first << i.second; + int n = 0; + for (auto & i : roots) { + // Obfuscate 'memory' roots as they exposes information about other users, + if (i.first.rfind("{memory:", 0) == 0) { + to << fmt("{memory:%d}", n++) << i.second; + } else { + to << i.first << i.second; + } + } break; } From ebc86550f92ec76cda0961ecc625944ec402d2cd Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 00:54:52 +0100 Subject: [PATCH 0051/2548] Make roots a map of store paths to pinning links This new structure makes more sense as there may be many sources rooting the same store path. Many profiles can reference the same path but this is even more true with /proc//maps where distinct pids can and often do map the same store path. This implementation is also more efficient as the `Roots` map contains only one entry per rooted store path. --- src/libstore/gc.cc | 39 +++++++++++++++++------------------- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.hh | 2 +- src/nix-daemon/nix-daemon.cc | 19 +++++++++++------- src/nix-store/nix-store.cc | 12 ++++++----- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 73630f36d..da6799f6f 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -130,7 +130,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, gcroots directory. */ if (settings.checkRootReachability) { Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) + if (roots[storePath].count(gcRoot) == 0) printError( format( "warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -266,7 +266,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); else printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); }; @@ -306,7 +306,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = storeDir + "/" + baseNameOf(path); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); } } @@ -346,10 +346,10 @@ Roots LocalStore::findRoots() FDs fds; pid_t prev = -1; size_t n = 0; - for (auto & root : readTempRoots(fds)) { - if (prev != root.first) n = 0; - prev = root.first; - roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second; + for (auto & [pid, root] : readTempRoots(fds)) { + if (prev != pid) n = 0; + prev = pid; + roots[root].emplace(fmt("{temp:%d:%d}", pid, n++)); } return roots; @@ -374,8 +374,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - roots.emplace((format("{memory:%1%") % file).str(), - std::string(static_cast(buf), res)); + roots[std::string(static_cast(buf), res)] + .emplace((format("{memory:%1%") % file).str()); return; } @@ -388,7 +388,7 @@ static string quoteRegexChars(const string & raw) static void readFileRoots(const char * path, Roots & roots) { try { - roots.emplace(path, readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; @@ -434,19 +434,17 @@ void LocalStore::findRuntimeRoots(Roots & roots) try { auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); - int n = 0; for (const auto& line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked.emplace((format("{memory:%1%:%2%}") % mapFile % n++).str(), match[1]); + unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str()); } auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; - n = 0; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked.emplace((format("{memory:%1%:%2%}") % envFile % n++).str(), i->str()); + unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str()); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -463,11 +461,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - int n = 0; for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked.emplace((format("{memory:%1%:%2%}" % LSOF % n++).str(), match[1]); + unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str()); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ @@ -480,12 +477,12 @@ void LocalStore::findRuntimeRoots(Roots & roots) readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & root : unchecked) { - if (isInStore(root.second)) { - Path path = toStorePath(root.second); + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.emplace(root.first, path); + roots[path].insert(links.begin(), links.end()); } } } @@ -757,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printError(format("finding garbage collector roots...")); Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); - for (auto & i : rootMap) state.roots.insert(i.second); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index def140cfb..a4dd28af0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -606,7 +606,7 @@ Roots RemoteStore::findRoots() while (count--) { Path link = readString(conn->from); Path target = readStorePath(*this, conn->from); - result[link] = target; + result[target].emplace(link); } return result; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ad0f8df11..b4e5f5511 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -47,7 +47,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map Roots; +typedef std::map> Roots; struct GCOptions diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index faa23b268..014378d27 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -477,14 +477,19 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); Roots roots = store->findRoots(); logger->stopWork(); - to << roots.size(); + size_t total_length = 0; + for (auto & root : roots) + total_length += root.second.size(); + to << total_length; int n = 0; - for (auto & i : roots) { - // Obfuscate 'memory' roots as they exposes information about other users, - if (i.first.rfind("{memory:", 0) == 0) { - to << fmt("{memory:%d}", n++) << i.second; - } else { - to << i.first << i.second; + for (auto & [target, links] : roots) { + for (auto & link : links) { + // Obfuscate 'memory' roots as they expose information about other users, + if (link.rfind("{memory:", 0) == 0) { + to << fmt("{memory:%d}", n++) << target; + } else { + to << link << target; + } } } break; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 33138baff..b281ea2dd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,9 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + for (auto & [path, roots] : roots) + if (referrers.find(path) != referrers.end()) + for (auto & root : roots) + cout << format("%1% -> %2%\n") % root % path; break; } @@ -591,8 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + for (auto & [path, roots] : roots) + for (auto & root : roots) + cout << root << " -> " << path << std::endl; } else { From 8574b703424b73f6f75fc549b1738a6534d6a01e Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:15:10 +0100 Subject: [PATCH 0052/2548] fixup! Make roots a map of store paths to pinning links --- src/nix-store/nix-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b281ea2dd..89831eb3d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & [path, roots] : roots) + for (auto & [path, links] : roots) if (referrers.find(path) != referrers.end()) - for (auto & root : roots) - cout << format("%1% -> %2%\n") % root % path; + for (auto & link : links) + cout << format("%1%\n") % link; break; } From fc02b1b3eee7632049040a96b056d769a088b2ea Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:16:25 +0100 Subject: [PATCH 0053/2548] Also print rooted path in `nix-store -q --roots` --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 89831eb3d..11d0d573f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -431,7 +431,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & [path, links] : roots) if (referrers.find(path) != referrers.end()) for (auto & link : links) - cout << format("%1%\n") % link; + cout << format("%1% -> %2%\n") % link % path; break; } From 5c56570726bc3c546b5cd24da96ceed23b9fdc90 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 1 Mar 2019 01:29:37 +0100 Subject: [PATCH 0054/2548] Also obfuscate the number of memory roots --- src/nix-daemon/nix-daemon.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 014378d27..63b5eb87d 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -478,15 +478,29 @@ static void performOp(TunnelLogger * logger, ref store, Roots roots = store->findRoots(); logger->stopWork(); size_t total_length = 0; - for (auto & root : roots) - total_length += root.second.size(); + bool hasMemoryLink; + for (auto & [target, links] : roots) { + hasMemoryLink = false; + for (auto & link : links) { + if (link.rfind("{memory:", 0) == 0) { + if (hasMemoryLink) continue; + ++total_length; + hasMemoryLink = true; + } else { + ++total_length; + } + } + } to << total_length; int n = 0; for (auto & [target, links] : roots) { + bool hasMemoryLink = false; for (auto & link : links) { // Obfuscate 'memory' roots as they expose information about other users, if (link.rfind("{memory:", 0) == 0) { + if (hasMemoryLink) continue; to << fmt("{memory:%d}", n++) << target; + hasMemoryLink = true; } else { to << link << target; } From 9d87e3fbd2c690f03eadbcada7ff8b880033155c Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sat, 9 Mar 2019 23:55:15 +0100 Subject: [PATCH 0055/2548] require c++ 17 This fixes warnings about "structured binding declaration" requiring "-std=c++1z". --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 410b20972..4eea008b0 100644 --- a/configure.ac +++ b/configure.ac @@ -62,7 +62,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_14 +AX_CXX_COMPILE_STDCXX_17 AC_CHECK_TOOL([AR], [ar]) From 9d7221183a6825d798267288ebe7d3e2adcd8cde Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:04:37 +0100 Subject: [PATCH 0056/2548] unify naming of roots as links & targets --- src/nix-store/nix-store.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 11d0d573f..b5530b39e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -428,10 +428,10 @@ static void opQuery(Strings opFlags, Strings opArgs) referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } Roots roots = store->findRoots(); - for (auto & [path, links] : roots) - if (referrers.find(path) != referrers.end()) + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << format("%1% -> %2%\n") % link % path; + cout << format("%1% -> %2%\n") % link % target; break; } @@ -592,9 +592,9 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(); - for (auto & [path, roots] : roots) - for (auto & root : roots) - cout << root << " -> " << path << std::endl; + for (auto & [target, links] : roots) + for (auto & link : links) + cout << link << " -> " << target << std::endl; } else { From 38ee16ae9c277b0512559282c950c48c71e27697 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:37:52 +0100 Subject: [PATCH 0057/2548] Unify internal findRootsXxx() api --- src/libstore/gc.cc | 35 +++++++++++++---------------------- src/libstore/local-store.hh | 4 ++-- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index da6799f6f..2a3140354 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -197,10 +197,8 @@ void LocalStore::addTempRoot(const Path & path) } -std::set> LocalStore::readTempRoots(FDs & fds) +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) { - std::set> tempRoots; - /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { @@ -250,14 +248,12 @@ std::set> LocalStore::readTempRoots(FDs & fds) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots.emplace(pid, root); + tempRoots[root].emplace(fmt("{temp:%d}", pid)); pos = end + 1; } fds.push_back(fd); /* keep open */ } - - return tempRoots; } @@ -321,10 +317,8 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -Roots LocalStore::findRootsNoTemp() +void LocalStore::findRootsNoTemp(Roots & roots) { - Roots roots; - /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); @@ -334,23 +328,16 @@ Roots LocalStore::findRootsNoTemp() to add running programs to the set of roots (to prevent them from being garbage collected). */ findRuntimeRoots(roots); - - return roots; } Roots LocalStore::findRoots() { - Roots roots = findRootsNoTemp(); + Roots roots; + findRootsNoTemp(roots); FDs fds; - pid_t prev = -1; - size_t n = 0; - for (auto & [pid, root] : readTempRoots(fds)) { - if (prev != pid) n = 0; - prev = pid; - roots[root].emplace(fmt("{temp:%d:%d}", pid, n++)); - } + findTempRoots(fds, roots); return roots; } @@ -752,7 +739,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap); for (auto & i : rootMap) state.roots.insert(i.first); @@ -760,8 +749,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - for (auto & root : readTempRoots(fds)) - state.tempRoots.insert(root.second); + Roots tempRoots; + findTempRoots(fds, tempRoots); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); /* After this point the set of roots or temporary roots cannot diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 730d4c917..39a34597e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,7 +180,7 @@ private: typedef std::shared_ptr FDPtr; typedef list FDs; - std::set> readTempRoots(FDs & fds); + void findTempRoots(FDs & fds, Roots & roots); public: @@ -267,7 +267,7 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - Roots findRootsNoTemp(); + void findRootsNoTemp(Roots & roots); void findRuntimeRoots(Roots & roots); From 4f4391193cca2f7a28d351947e12f954d9432693 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 00:45:55 +0100 Subject: [PATCH 0058/2548] Fix warning about unused variable --- src/nix-daemon/nix-daemon.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 63b5eb87d..0feafb013 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -477,11 +477,13 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); Roots roots = store->findRoots(); logger->stopWork(); + + // Pre-compute roots length using same algo as below. size_t total_length = 0; bool hasMemoryLink; - for (auto & [target, links] : roots) { + for (auto & root : roots) { hasMemoryLink = false; - for (auto & link : links) { + for (auto & link : root.second) { if (link.rfind("{memory:", 0) == 0) { if (hasMemoryLink) continue; ++total_length; @@ -491,6 +493,7 @@ static void performOp(TunnelLogger * logger, ref store, } } } + to << total_length; int n = 0; for (auto & [target, links] : roots) { From 115e2c8c67a949d3fa78fda450186803cd713958 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Sun, 10 Mar 2019 01:50:33 +0100 Subject: [PATCH 0059/2548] Update tests to the new --roots format --- tests/gc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gc.sh b/tests/gc.sh index 0adb05bf1..8b4f8d282 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -7,7 +7,7 @@ outPath=$(nix-store -rvv "$drvPath") rm -f "$NIX_STATE_DIR"/gcroots/foo ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo -[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR"/gcroots/foo ] +[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] nix-store --gc --print-roots | grep $outPath nix-store --gc --print-live | grep $outPath From a3f37d87eabcfb5dc581abcfa46e5e7d387dfa8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:27:16 +0100 Subject: [PATCH 0060/2548] findRuntimeRoots: Simplify/fix handling of /proc files Scanning of /proc//{exe,cwd} was broken because '{memory:' was prepended twice. Also, get rid of the whole '{memory:...}' thing because it's unnecessary, we can just list the file in /proc directly. --- src/libstore/gc.cc | 26 ++++++++++++-------------- src/libstore/store-api.hh | 3 ++- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 2a3140354..8c3607f66 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -362,8 +362,7 @@ try_again: } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] - .emplace((format("{memory:%1%") % file).str()); - return; + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -395,10 +394,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("{memory:/proc/%1%/exe}") % ent->d_name).str(), unchecked); - readProcLink((format("{memory:/proc/%1%/cwd}") % ent->d_name).str(), unchecked); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) @@ -407,9 +406,8 @@ void LocalStore::findRuntimeRoots(Roots & roots) } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), unchecked); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } if (errno) { if (errno == ESRCH) @@ -419,19 +417,19 @@ void LocalStore::findRuntimeRoots(Roots & roots) fdDir.reset(); try { - auto mapFile = (format("/proc/%1%/maps") % ent->d_name).str(); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); - for (const auto& line : mapLines) { + for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - unchecked[match[1]].emplace((format("{memory:%1%}") % mapFile).str()); + unchecked[match[1]].emplace(mapFile); } - auto envFile = (format("/proc/%1%/environ") % ent->d_name).str(); + auto envFile = fmt("/proc/%s/environ", ent->d_name); auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - unchecked[i->str()].emplace((format("{memory:%1%}") % envFile).str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -451,7 +449,7 @@ void LocalStore::findRuntimeRoots(Roots & roots) for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace((format("{memory:%1%}" % LSOF).str()); + unchecked[match[1]].emplace("{lsof}"); } } catch (ExecError & e) { /* lsof not installed, lsof failed */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b4e5f5511..bb1665976 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -47,7 +48,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map> Roots; +typedef std::unordered_map> Roots; struct GCOptions From 53522cb6ac19bd1da35a657988231cce9387be9c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:50:07 +0100 Subject: [PATCH 0061/2548] findRoots(): Add 'censor' parameter This is less brittle than filtering paths after the fact in nix-daemon. --- src/libstore/gc.cc | 30 +++++++++++++++----------- src/libstore/local-store.hh | 8 +++---- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 6 ++++-- src/nix-daemon/nix-daemon.cc | 41 +++++++++--------------------------- src/nix-store/nix-store.cc | 4 ++-- 7 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8c3607f66..d8a5da0d4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -129,7 +129,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); + Roots roots = findRoots(false); if (roots[storePath].count(gcRoot) == 0) printError( format( @@ -197,7 +197,10 @@ void LocalStore::addTempRoot(const Path & path) } -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) +static std::string censored = "{censored}"; + + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -248,7 +251,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots[root].emplace(fmt("{temp:%d}", pid)); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } @@ -317,7 +320,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -void LocalStore::findRootsNoTemp(Roots & roots) +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); @@ -327,17 +330,17 @@ void LocalStore::findRootsNoTemp(Roots & roots) NIX_ROOT_FINDER environment variable. This is typically used to add running programs to the set of roots (to prevent them from being garbage collected). */ - findRuntimeRoots(roots); + findRuntimeRoots(roots, censor); } -Roots LocalStore::findRoots() +Roots LocalStore::findRoots(bool censor) { Roots roots; - findRootsNoTemp(roots); + findRootsNoTemp(roots, censor); FDs fds; - findTempRoots(fds, roots); + findTempRoots(fds, roots, censor); return roots; } @@ -381,7 +384,7 @@ static void readFileRoots(const char * path, Roots & roots) } } -void LocalStore::findRuntimeRoots(Roots & roots) +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { Roots unchecked; @@ -467,7 +470,10 @@ void LocalStore::findRuntimeRoots(Roots & roots) Path path = toStorePath(target); if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots[path].insert(links.begin(), links.end()); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } } @@ -739,7 +745,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printError(format("finding garbage collector roots...")); Roots rootMap; if (!options.ignoreLiveness) - findRootsNoTemp(rootMap); + findRootsNoTemp(rootMap, true); for (auto & i : rootMap) state.roots.insert(i.first); @@ -748,7 +754,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) can be added to the set of temporary roots. */ FDs fds; Roots tempRoots; - findTempRoots(fds, tempRoots); + findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 39a34597e..6b655647b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,11 +180,11 @@ private: typedef std::shared_ptr FDPtr; typedef list FDs; - void findTempRoots(FDs & fds, Roots & roots); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -267,9 +267,9 @@ private: void findRoots(const Path & path, unsigned char type, Roots & roots); - void findRootsNoTemp(Roots & roots); + void findRootsNoTemp(Roots & roots, bool censor); - void findRuntimeRoots(Roots & roots); + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a4dd28af0..15faf78a5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -596,7 +596,7 @@ void RemoteStore::syncWithGC() } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); conn->to << wopFindRoots; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4f554b598..e5ab79e59 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -82,7 +82,7 @@ public: void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bb1665976..40887b6aa 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -484,8 +484,10 @@ public: /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) { unsupported("findRoots"); } /* Perform a garbage collection. */ diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 0feafb013..dd8b9bd1e 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -475,40 +475,19 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(); + Roots roots = store->findRoots(true); logger->stopWork(); - // Pre-compute roots length using same algo as below. - size_t total_length = 0; - bool hasMemoryLink; - for (auto & root : roots) { - hasMemoryLink = false; - for (auto & link : root.second) { - if (link.rfind("{memory:", 0) == 0) { - if (hasMemoryLink) continue; - ++total_length; - hasMemoryLink = true; - } else { - ++total_length; - } - } - } + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; - to << total_length; - int n = 0; - for (auto & [target, links] : roots) { - bool hasMemoryLink = false; - for (auto & link : links) { - // Obfuscate 'memory' roots as they expose information about other users, - if (link.rfind("{memory:", 0) == 0) { - if (hasMemoryLink) continue; - to << fmt("{memory:%d}", n++) << target; - hasMemoryLink = true; - } else { - to << link << target; - } - } - } break; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b5530b39e..6f315e509 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -427,7 +427,7 @@ static void opQuery(Strings opFlags, Strings opArgs) maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } - Roots roots = store->findRoots(); + Roots roots = store->findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) @@ -591,7 +591,7 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); + Roots roots = store->findRoots(false); for (auto & [target, links] : roots) for (auto & link : links) cout << link << " -> " << target << std::endl; From 578ed7a259ddf7a85c85fe4e0200179ae908171e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:52:13 +0100 Subject: [PATCH 0062/2548] findRoots(): Don't censor for trusted users They're pretty much root anyway. --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index dd8b9bd1e..8d63b8f36 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -475,7 +475,7 @@ static void performOp(TunnelLogger * logger, ref store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(true); + Roots roots = store->findRoots(!trusted); logger->stopWork(); size_t size = 0; From 86f3b94c8c28defd7402245d60af15ce429986ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Mar 2019 13:59:10 +0100 Subject: [PATCH 0063/2548] nix-store --gc --print-roots: Sort output --- src/nix-store/nix-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6f315e509..f324056bb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -592,9 +592,13 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(false); + std::set> roots2; + // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) - cout << link << " -> " << target << std::endl; + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; } else { From ef52ccf035fe0bcd78226891022171a28dd6fcbe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:43:32 +0100 Subject: [PATCH 0064/2548] experimental/optional -> optional --- src/libexpr/eval.hh | 2 +- src/libexpr/get-drvs.cc | 2 +- src/libexpr/get-drvs.hh | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchGit.cc | 4 ++-- src/libstore/build.cc | 10 +++++----- src/libstore/parsed-derivations.cc | 4 ++-- src/libstore/parsed-derivations.hh | 8 ++++---- src/libstore/remote-store.hh | 2 +- src/libutil/lru-cache.hh | 4 ++-- src/libutil/serialise.cc | 2 +- src/libutil/util.cc | 2 +- src/libutil/util.hh | 6 +++--- src/nix/add-to-store.cc | 2 +- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9fe387891..a314e01e0 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -81,7 +81,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional allowedPaths; + std::optional allowedPaths; Value vEmptySet; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3..21a4d7917 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional getDerivation(EvalState & state, Value & v, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index daaa635fe..d7860fc6a 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -78,7 +78,7 @@ typedef list DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional getDerivation(EvalState & state, +std::optional getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0da9f702f..6b0c55e72 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -555,7 +555,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional outputHash; + std::optional outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b46d2f258..aaf02c856 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -26,7 +26,7 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, - std::experimental::optional ref, std::string rev, + std::optional ref, std::string rev, const std::string & name) { if (evalSettings.pureEval && rev == "") @@ -190,7 +190,7 @@ GitInfo exportGit(ref store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional ref; + std::optional ref; std::string rev; std::string name = "source"; PathSet context; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 47ee8b48f..6b88b1307 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2413,7 +2413,7 @@ void DerivationGoal::writeStructuredAttrs() objects consisting entirely of those values. (So nested arrays or objects are not supported.) */ - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional { + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { if (value.is_string()) return shellEscape(value); @@ -3311,8 +3311,8 @@ void DerivationGoal::checkOutputs(const std::map & outputs) struct Checks { bool ignoreSelfRefs = false; - std::experimental::optional maxSize, maxClosureSize; - std::experimental::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; }; /* Compute the closure and closure size of some output. This @@ -3359,7 +3359,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) info.path, closureSize, *checks.maxClosureSize); } - auto checkRefs = [&](const std::experimental::optional & value, bool allowed, bool recursive) + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) { if (!value) return; @@ -3413,7 +3413,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) if (maxClosureSize != output->end()) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::experimental::optional { + auto get = [&](const std::string & name) -> std::optional { auto i = output->find(name); if (i != output->end()) { Strings res; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index dc3286482..17fde00a0 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -16,7 +16,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) } } -std::experimental::optional ParsedDerivation::getStringAttr(const std::string & name) const +std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); @@ -56,7 +56,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const } } -std::experimental::optional ParsedDerivation::getStringsAttr(const std::string & name) const +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const { if (structuredAttrs) { auto i = structuredAttrs->find(name); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 0a82c1461..ed07dc652 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -8,22 +8,22 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::experimental::optional structuredAttrs; + std::optional structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::experimental::optional & getStructuredAttrs() const + const std::optional & getStructuredAttrs() const { return structuredAttrs; } - std::experimental::optional getStringAttr(const std::string & name) const; + std::optional getStringAttr(const std::string & name) const; bool getBoolAttr(const std::string & name, bool def = false) const; - std::experimental::optional getStringsAttr(const std::string & name) const; + std::optional getStringsAttr(const std::string & name) const; StringSet getRequiredSystemFeatures() const; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index e5ab79e59..80f18ab71 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -149,7 +149,7 @@ public: private: ref openConnection() override; - std::experimental::optional path; + std::optional path; }; diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e63..8b83f842c 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include #include -#include +#include namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional get(const Key & key) + std::optional get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0e75eeec2..8201549fd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,7 +171,7 @@ std::unique_ptr sinkToSource( std::function fun; std::function eof; - std::experimental::optional coro; + std::optional coro; bool started = false; SinkToSource(std::function fun, std::function eof) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7eca35577..e3dcd246c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -965,7 +965,7 @@ std::vector stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional & input) + const std::optional & input) { RunOptions opts(program, args); opts.searchPath = searchPath; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bda87bee4..9f239bff3 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -259,14 +259,14 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional & input = {}); + const std::optional & input = {}); struct RunOptions { Path program; bool searchPath = true; Strings args; - std::experimental::optional input; + std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; bool _killStderr = false; diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d..e86b96e3f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional namePart; + std::optional namePart; CmdAddToStore() { From 1f64f4c7c83d8efd67051a399c738c71815e0594 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 12 Feb 2019 13:44:20 +0100 Subject: [PATCH 0065/2548] pkg-config files: Use c++17 --- src/libexpr/nix-expr.pc.in | 2 +- src/libmain/nix-main.pc.in | 2 +- src/libstore/nix-store.pc.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f45..80f7a492b 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c48..37b03dcd4 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faad..6d67b1e03 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 From 34fade478a84d822646a8daa334014134c0ec59e Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 14 Mar 2019 23:51:18 +0100 Subject: [PATCH 0066/2548] docs: fix typo in AllowDirectReads It was just missing an `l` but should be fixed anyway. --- doc/manual/packages/s3-substituter.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index 2ec9687a0..e7589ffdb 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -89,7 +89,7 @@ the S3 URL: "Version": "2012-10-17", "Statement": [ { - "Sid": "AlowDirectReads", + "Sid": "AllowDirectReads", "Action": [ "s3:GetObject", "s3:GetBucketLocation" From 81a23fa7e2eb4d58d5edf2a145af5c8bbe4c331e Mon Sep 17 00:00:00 2001 From: Chaker Benhamed Date: Fri, 15 Mar 2019 13:16:20 +0100 Subject: [PATCH 0067/2548] Add a 5 seconds of timeouts to connect to S3. The default is 1000ms, but we can hit it a lot of we don't have direct link to AWS (e.g. using VPN). --- src/libstore/s3-binary-cache-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 51de89e0d..cd547a964 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -126,6 +126,7 @@ ref S3Helper::makeConfig(const string & region res->endpointOverride = endpoint; } res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; res->retryStrategy = std::make_shared(); res->caFile = settings.caFile; return res; From 684c7fff80b575da8055015c6b87693fd1d858ca Mon Sep 17 00:00:00 2001 From: Joachim Breitner Date: Sat, 16 Mar 2019 19:03:03 +0100 Subject: [PATCH 0068/2548] Link to `builders-use-substitutes` in chapter on distributed builds fixes #2730. --- doc/manual/advanced-topics/distributed-builds.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index bbb573e35..ce2e077ed 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -180,4 +180,8 @@ builders = @/etc/nix/machines causes the list of machines in /etc/nix/machines to be included. (This is the default.) +If you want the builders to use caches, you likely want to set +the option builders-use-substitutes +in your local nix.conf. + From 6f093073b6ff38fb2c4e22331e73930baa1b30e7 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 21 Mar 2019 09:35:43 +0100 Subject: [PATCH 0069/2548] manual: include builtins.* for globally available builtins This improves searchability. --- doc/manual/expressions/builtins.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 22d998bd0..cefc8b78e 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -23,6 +23,7 @@ available as builtins.derivation. abort s + builtins.abort s Abort Nix expression evaluation, print error message s. @@ -251,6 +252,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" derivation attrs + builtins.derivation + attrs derivation is described in . @@ -260,6 +263,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" dirOf s + builtins.dirOf s Return the directory part of the string s, that is, everything before the final @@ -318,6 +322,8 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" fetchTarball url + builtins.fetchTarball + url Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive @@ -714,6 +720,8 @@ builtins.genList (x: x * x) 5 import path + builtins.import + path Load, parse and return the Nix expression in the file path. If path @@ -857,6 +865,8 @@ x: x + 456 isNull e + builtins.isNull + e Return true if e evaluates to null, @@ -925,6 +935,8 @@ builtins.listToAttrs map f list + builtins.map + f list Apply the function f to each element in the list list. For @@ -1119,6 +1131,8 @@ Evaluates to [ "foo" ]. removeAttrs set list + builtins.removeAttrs + set list Remove the attributes listed in list from @@ -1287,6 +1301,8 @@ builtins.substring 0 3 "nixos" throw s + builtins.throw + s Throw an error message s. This usually aborts Nix expression @@ -1405,6 +1421,7 @@ in foo toString e + builtins.toString e Convert the expression e to a string. From 2aa89daab3a8351a03f6d57d3e1f737cccd59850 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 21 Mar 2019 14:32:20 +0100 Subject: [PATCH 0070/2548] eval: improve type description for primops and applied primops This can make type errors a little easier to understand. --- src/libexpr/eval.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2a194d0e0..211f7a55f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -130,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { @@ -144,8 +154,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } From d4ee8afd59cd7935f59b730c432cf58460af8a84 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Feb 2019 06:53:01 +0100 Subject: [PATCH 0071/2548] Implemented --flake flag for nix build Also fixed Eelco's PR comments --- src/libexpr/primops/flake.cc | 64 +++++++++++++++++++++++++++++---- src/libexpr/primops/flake.hh | 11 ++++-- src/libexpr/primops/flakeref.hh | 3 ++ src/libutil/util.cc | 1 - src/nix/build.cc | 15 ++++++++ src/nix/command.cc | 9 ----- src/nix/command.hh | 7 ---- src/nix/flake.cc | 29 +++++++++++++-- 8 files changed, 111 insertions(+), 28 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 9d1da84f1..df0845c24 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -3,7 +3,9 @@ #include "eval-inline.hh" #include "fetchGit.hh" #include "download.hh" +#include "args.hh" +#include #include #include #include @@ -32,10 +34,10 @@ static std::unique_ptr readRegistry(const Path & path) } /* Write the registry or lock file to a file. */ -static void writeRegistry(FlakeRegistry registry, Path path = "./flake.lock") +void writeRegistry(FlakeRegistry registry, Path path) { nlohmann::json json = {}; - json["value"] = 0; // Not sure whether this should be 0. + json["version"] = 1; json["flakes"] = {}; for (auto elem : registry.entries) { json["flakes"][elem.first] = elem.second.ref.to_string(); @@ -107,9 +109,21 @@ struct FlakeSourceInfo static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) { - assert(flakeRef.isDirect()); + FlakeRef directFlakeRef = FlakeRef(flakeRef); + if (!flakeRef.isDirect()) + { + std::vector registries; + // 'pureEval' is a setting which cannot be changed in `nix flake`, + // but without flagging it off, we can't use any FlakeIds. + // if (!evalSettings.pureEval) { + registries.push_back(&state.getFlakeRegistry()); + // } + directFlakeRef = lookupFlake(state, flakeRef, registries); + } + assert(directFlakeRef.isDirect()); + // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes.. - if (auto refData = std::get_if(&flakeRef.data)) { + if (auto refData = std::get_if(&directFlakeRef.data)) { // FIXME: require hash in pure mode. // FIXME: use regular /archive URLs instead? api.github.com @@ -141,7 +155,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) return info; } - else if (auto refData = std::get_if(&flakeRef.data)) { + else if (auto refData = std::get_if(&directFlakeRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, refData->ref, refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); FlakeSourceInfo info; @@ -165,7 +179,16 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (state.allowedPaths) state.allowedPaths->insert(flakePath); - Flake flake; + FlakeRef newFlakeRef(flakeRef); + if (std::get_if(&newFlakeRef.data)) { + FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef); + if (srcInfo.rev) { + std::string uri = flakeRef.to_string(); + newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string()); + } + } + + Flake flake(newFlakeRef); Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -258,6 +281,35 @@ static std::tuple> resolveFlake(EvalState & st return {*topFlakeId, std::move(done)}; } +FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) +{ + FlakeRegistry newLockFile; + std::map myDependencyMap = get<1>(resolveFlake(evalState, flakeRef, false)); + // Nick assumed that "topRefPure" means that the Flake for flakeRef can be + // fetched purely. + for (auto const& require : myDependencyMap) { + FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.second.ref); + // The FlakeRefs are immutable because they come out of the Flake objects, + // not from the requires. + newLockFile.entries.insert(std::pair(require.first, entry)); + } + return newLockFile; +} + +void updateLockFile(EvalState & state, std::string path) +{ + // 'path' is the path to the local flake repo. + FlakeRef flakeRef = FlakeRef(path); + if (std::get_if(&flakeRef.data)) { + FlakeRegistry newLockFile = updateLockFile(state, flakeRef); + writeRegistry(newLockFile, path + "/flake.lock"); + } else if (std::get_if(&flakeRef.data)) { + throw UsageError("You can only update local flakes, not flakes on GitHub."); + } else { + throw UsageError("You can only update local flakes, not flakes through their FlakeId."); + } +} + Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v) { // FIXME: temporary hack to make the default installation source diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 90c6bc7d2..b3a755311 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -13,6 +13,7 @@ struct FlakeRegistry struct Entry { FlakeRef ref; + Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; }; std::map entries; }; @@ -21,9 +22,12 @@ Value * makeFlakeRegistryValue(EvalState & state); Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); +void writeRegistry(FlakeRegistry, Path); + struct Flake { FlakeId id; + FlakeRef ref; std::string description; Path path; std::vector requires; @@ -32,9 +36,12 @@ struct Flake // commit hash // date // content hash + Flake(FlakeRef & flakeRef) : ref(flakeRef) {}; }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef); +Flake getFlake(EvalState &, const FlakeRef &); -void writeRegistry(FlakeRegistry); +FlakeRegistry updateLockFile(EvalState &, Flake &); + +void updateLockFile(EvalState &, std::string); } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index ad0cf8630..4d1756b49 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -129,6 +129,9 @@ struct FlakeRef // Parse a flake URI. FlakeRef(const std::string & uri); + // Default constructor + FlakeRef(const FlakeRef & flakeRef) : data(flakeRef.data) {}; + /* Unify two flake references so that the resulting reference combines the information from both. For example, "nixpkgs/" and "github:NixOS/nixpkgs" unifies to diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e3dcd246c..b0a2b853e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -344,7 +344,6 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } - string readLine(int fd) { string s; diff --git a/src/nix/build.cc b/src/nix/build.cc index b329ac38a..12ef08679 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,3 +1,5 @@ +#include "primops/flake.hh" +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -9,6 +11,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; + std::optional flakeUri = std::nullopt; + CmdBuild() { mkFlag() @@ -22,6 +26,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand .longName("no-link") .description("do not create a symlink to the build result") .set(&outLink, Path("")); + + mkFlag() + .longName("flake") + .description("update lock file of given flake") + .dest(&flakeUri); } std::string name() override @@ -52,6 +61,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand { auto buildables = build(store, dryRun ? DryRun : Build, installables); + auto evalState = std::make_shared(searchPath, store); + if (dryRun) return; for (size_t i = 0; i < buildables.size(); ++i) { @@ -66,6 +77,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand store2->addPermRoot(output.second, absPath(symlink), true); } } + + if (flakeUri) { + updateLockFile(*evalState, *flakeUri); + } } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index e1e32aaae..5967ab36c 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -27,15 +27,6 @@ void StoreCommand::run() run(getStore()); } -JsonFormattable::JsonFormattable() -{ - mkFlag() - .longName("json-formattable") - .shortName('j') - .description("output will be printed as json") - .handler([&]() { jsonFormatting = true; }); -} - StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { diff --git a/src/nix/command.hh b/src/nix/command.hh index 5c2f8c304..b3248222e 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -26,13 +26,6 @@ private: std::shared_ptr _store; }; -struct JsonFormattable : virtual Command -{ - bool jsonFormatting = false;; - - JsonFormattable(); -}; - struct Buildable { Path drvPath; // may be empty diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 22e5b297c..6cef38936 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -34,7 +34,28 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; -struct CmdFlakeInfo : FlakeCommand, JsonFormattable +struct CmdFlakeUpdate : FlakeCommand +{ + std::string name() override + { + return "update"; + } + + std::string description() override + { + return "update flake lock file"; + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + if (flakeUri == "") flakeUri = absPath("./flake.nix"); + updateLockFile(*evalState, flakeUri); + } +}; + +struct CmdFlakeInfo : FlakeCommand, MixJSON { std::string name() override { @@ -50,7 +71,7 @@ struct CmdFlakeInfo : FlakeCommand, JsonFormattable { auto evalState = std::make_shared(searchPath, store); nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); - if (jsonFormatting) { + if (json) { nlohmann::json j; j["location"] = flake.path; j["description"] = flake.description; @@ -65,7 +86,9 @@ struct CmdFlakeInfo : FlakeCommand, JsonFormattable struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() - : MultiCommand({make_ref(), make_ref()}) + : MultiCommand({make_ref() + , make_ref() + , make_ref()}) { } From e007f367bd605ad14ddf84d1d5ad611aa427d338 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Feb 2019 06:53:01 +0100 Subject: [PATCH 0072/2548] Fixed minor things --- src/libexpr/primops/flake.cc | 6 +++--- src/nix/build.cc | 13 ++++++------- src/nix/command.hh | 21 +++++++++++++++------ src/nix/flake.cc | 11 ++++++++--- src/nix/installables.cc | 1 + 5 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index df0845c24..48a036875 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -299,14 +299,14 @@ FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) void updateLockFile(EvalState & state, std::string path) { // 'path' is the path to the local flake repo. - FlakeRef flakeRef = FlakeRef(path); + FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { FlakeRegistry newLockFile = updateLockFile(state, flakeRef); writeRegistry(newLockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { - throw UsageError("You can only update local flakes, not flakes on GitHub."); + throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { - throw UsageError("You can only update local flakes, not flakes through their FlakeId."); + throw UsageError("you can only update local flakes, not flakes through their FlakeId"); } } diff --git a/src/nix/build.cc b/src/nix/build.cc index 12ef08679..5ab22e26c 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,7 +11,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; - std::optional flakeUri = std::nullopt; + std::optional gitRepo = std::nullopt; CmdBuild() { @@ -28,9 +28,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand .set(&outLink, Path("")); mkFlag() - .longName("flake") - .description("update lock file of given flake") - .dest(&flakeUri); + .longName("update-lock-file") + .description("update the lock file") + .dest(&gitRepo); } std::string name() override @@ -78,9 +78,8 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - if (flakeUri) { - updateLockFile(*evalState, *flakeUri); - } + if(gitRepo) + updateLockFile(*evalState, *gitRepo); } }; diff --git a/src/nix/command.hh b/src/nix/command.hh index b3248222e..c58d5d56e 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -34,15 +34,24 @@ struct Buildable typedef std::vector Buildables; +struct GitRepoCommand : virtual Args +{ + std::string gitPath = absPath("."); + + GitRepoCommand () + { + expectArg("git-path", &gitPath, true); + } +}; + struct FlakeCommand : virtual Args, StoreCommand, MixEvalArgs { - std::string flakeUri; + std::string flakeUri; -public: - FlakeCommand() - { - expectArg("flake-uri", &flakeUri); - } + FlakeCommand() + { + expectArg("flake-uri", &flakeUri); + } }; struct Installable diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6cef38936..a5a1d34db 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -34,7 +34,7 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; -struct CmdFlakeUpdate : FlakeCommand +struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs { std::string name() override { @@ -51,7 +51,12 @@ struct CmdFlakeUpdate : FlakeCommand auto evalState = std::make_shared(searchPath, store); if (flakeUri == "") flakeUri = absPath("./flake.nix"); - updateLockFile(*evalState, flakeUri); + int result = updateLockFile(*evalState, flakeUri); + if (result == 1) { + std::cout << "You can only update local flakes, not flakes on GitHub.\n"; + } else if (result == 2) { + std::cout << "You can only update local flakes, not flakes through their FlakeId.\n"; + } } }; @@ -77,8 +82,8 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON j["description"] = flake.description; std::cout << j.dump(4) << std::endl; } else { - std::cout << "Location: " << flake.path << "\n"; std::cout << "Description: " << flake.description << "\n"; + std::cout << "Location: " << flake.path << "\n"; } } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0453c72c2..21e9e73b8 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -234,6 +234,7 @@ Buildables build(ref store, RealiseMode mode, PathSet pathsToBuild; for (auto & i : installables) { + std::cout << i->what() << std::endl; for (auto & b : i->toBuildables()) { if (b.drvPath != "") { StringSet outputNames; From 5e4d92d267c080bcb81168e37429bbb56bc39fb2 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Sun, 10 Mar 2019 07:05:05 +0100 Subject: [PATCH 0073/2548] Issue #15 is finished --- src/libexpr/primops/flake.cc | 13 ++-- src/libexpr/primops/flake.hh | 5 ++ src/libexpr/primops/flakeref.cc | 15 +++++ src/libexpr/primops/flakeref.hh | 3 +- src/nix/command.hh | 2 +- src/nix/flake.cc | 113 +++++++++++++++++++++++++++++--- 6 files changed, 136 insertions(+), 15 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 48a036875..b74e0b4b7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -12,9 +12,14 @@ namespace nix { +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + /* Read the registry or a lock file. (Currently they have an identical format. */ -static std::unique_ptr readRegistry(const Path & path) +std::unique_ptr readRegistry(const Path & path) { auto registry = std::make_unique(); @@ -40,7 +45,7 @@ void writeRegistry(FlakeRegistry registry, Path path) json["version"] = 1; json["flakes"] = {}; for (auto elem : registry.entries) { - json["flakes"][elem.first] = elem.second.ref.to_string(); + json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; } writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } @@ -183,8 +188,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (std::get_if(&newFlakeRef.data)) { FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef); if (srcInfo.rev) { - std::string uri = flakeRef.to_string(); - newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string()); + std::string uri = flakeRef.baseRef().to_string(); + newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string(Base16, false)); } } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index b3a755311..4e49becc7 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -14,14 +14,19 @@ struct FlakeRegistry { FlakeRef ref; Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; + Entry operator=(const Entry & entry) { return Entry(entry.ref); } }; std::map entries; }; +Path getUserRegistryPath(); + Value * makeFlakeRegistryValue(EvalState & state); Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); +std::unique_ptr readRegistry(const Path &); + void writeRegistry(FlakeRegistry, Path); struct Flake diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index a2700f102..8e7c1f8df 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -152,4 +152,19 @@ bool FlakeRef::isImmutable() const else abort(); } +FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. +{ + FlakeRef result(*this); + if (auto refData = std::get_if(&result.data)) { + refData->ref = std::nullopt; + refData->rev = std::nullopt; + } else if (auto refData = std::get_if(&result.data)) { + refData->ref = std::nullopt; + refData->rev = std::nullopt; + } else if (auto refData = std::get_if(&result.data)) { + refData->ref = std::nullopt; + refData->rev = std::nullopt; + } + return result; +} } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 4d1756b49..fb365e101 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -153,6 +153,7 @@ struct FlakeRef /* Check whether this is an "immutable" flake reference, that is, one that contains a commit hash or content hash. */ bool isImmutable() const; -}; + FlakeRef baseRef() const; +}; } diff --git a/src/nix/command.hh b/src/nix/command.hh index c58d5d56e..ffe64ccb7 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -44,7 +44,7 @@ struct GitRepoCommand : virtual Args } }; -struct FlakeCommand : virtual Args, StoreCommand, MixEvalArgs +struct FlakeCommand : virtual Args { std::string flakeUri; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a5a1d34db..fda903944 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -50,17 +50,12 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs { auto evalState = std::make_shared(searchPath, store); - if (flakeUri == "") flakeUri = absPath("./flake.nix"); - int result = updateLockFile(*evalState, flakeUri); - if (result == 1) { - std::cout << "You can only update local flakes, not flakes on GitHub.\n"; - } else if (result == 2) { - std::cout << "You can only update local flakes, not flakes through their FlakeId.\n"; - } + if (gitPath == "") gitPath = absPath("."); + updateLockFile(*evalState, gitPath); } }; -struct CmdFlakeInfo : FlakeCommand, MixJSON +struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand { std::string name() override { @@ -88,12 +83,112 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON } }; +struct CmdFlakeAdd : MixEvalArgs, Command +{ + std::string flakeId; + std::string flakeUri; + + std::string name() override + { + return "add"; + } + + std::string description() override + { + return "upsert flake in user flake registry"; + } + + CmdFlakeAdd() + { + expectArg("flake-id", &flakeId); + expectArg("flake-uri", &flakeUri); + } + + void run() override + { + FlakeRef newFlakeRef(flakeUri); + Path userRegistryPath = getUserRegistryPath(); + auto userRegistry = readRegistry(userRegistryPath); + FlakeRegistry::Entry entry(newFlakeRef); + userRegistry->entries.erase(flakeId); + userRegistry->entries.insert_or_assign(flakeId, newFlakeRef); + writeRegistry(*userRegistry, userRegistryPath); + } +}; + +struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command +{ + std::string flakeId; + + std::string name() override + { + return "remove"; + } + + std::string description() override + { + return "remove flake from user flake registry"; + } + + CmdFlakeRemove() + { + expectArg("flake-id", &flakeId); + } + + void run() override + { + Path userRegistryPath = getUserRegistryPath(); + auto userRegistry = readRegistry(userRegistryPath); + userRegistry->entries.erase(flakeId); + writeRegistry(*userRegistry, userRegistryPath); + } +}; + +struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs +{ + std::string flakeId; + + std::string name() override + { + return "pin"; + } + + std::string description() override + { + return "pin flake require in user flake registry"; + } + + CmdFlakePin() + { + expectArg("flake-id", &flakeId); + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + Path userRegistryPath = getUserRegistryPath(); + FlakeRegistry userRegistry = *readRegistry(userRegistryPath); + auto it = userRegistry.entries.find(flakeId); + if (it != userRegistry.entries.end()) { + FlakeRef oldRef = it->second.ref; + it->second.ref = getFlake(*evalState, oldRef).ref; + // The 'ref' in 'flake' is immutable. + writeRegistry(userRegistry, userRegistryPath); + } else + throw Error("the flake identifier '%s' does not exist in the user registry", flakeId); + } +}; + struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() : MultiCommand({make_ref() + , make_ref() , make_ref() - , make_ref()}) + , make_ref() + , make_ref() + , make_ref()}) { } From a554f523db34a5d6a8281c5228acfc128a8bd589 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0074/2548] Combining registries properly --- src/libexpr/eval.hh | 4 +- src/libexpr/primops/flake.cc | 104 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 4 +- src/nix/flake.cc | 8 ++- 4 files changed, 73 insertions(+), 47 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 35c01b97a..f6c894cad 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -316,10 +316,10 @@ private: public: - const FlakeRegistry & getFlakeRegistry(); + const std::vector> getFlakeRegistries(); private: - std::unique_ptr _flakeRegistry; + std::shared_ptr _flakeRegistry; std::once_flag _flakeRegistryInit; }; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index b74e0b4b7..9a528ce82 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -12,16 +12,11 @@ namespace nix { -Path getUserRegistryPath() -{ - return getHome() + "/.config/nix/registry.json"; -} - /* Read the registry or a lock file. (Currently they have an identical format. */ -std::unique_ptr readRegistry(const Path & path) +std::shared_ptr readRegistry(const Path & path) { - auto registry = std::make_unique(); + auto registry = std::make_shared(); auto json = nlohmann::json::parse(readFile(path)); @@ -50,37 +45,71 @@ void writeRegistry(FlakeRegistry registry, Path path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -const FlakeRegistry & EvalState::getFlakeRegistry() +Path getUserRegistryPath() { - std::call_once(_flakeRegistryInit, [&]() - { -#if 0 - auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json"; + return getHome() + "/.config/nix/registry.json"; +} - auto registryFile = getDownloader()->download(DownloadRequest(registryUri)); -#endif +std::shared_ptr getGlobalRegistry() +{ + // TODO: Make a global registry, and return it here. + return std::make_shared(); +} - auto registryFile = settings.nixDataDir + "/nix/flake-registry.json"; +std::shared_ptr getUserRegistry() +{ + return readRegistry(getUserRegistryPath()); +} - _flakeRegistry = readRegistry(registryFile); - }); +// Project-specific registry saved in flake-registry.json. +std::shared_ptr getLocalRegistry() +{ + Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; + return readRegistry(registryFile); +} - return *_flakeRegistry; +std::shared_ptr getFlagRegistry() +{ + return std::make_shared(); + // TODO: Implement this once the right flags are implemented. +} + +// This always returns a vector with globalReg, userReg, localReg, flakeReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const std::vector> EvalState::getFlakeRegistries() +{ + std::vector> registries; + if (!evalSettings.pureEval) { + registries.push_back(std::make_shared()); // global + registries.push_back(std::make_shared()); // user + registries.push_back(std::make_shared()); // local + } else { + registries.push_back(getGlobalRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getLocalRegistry()); + } + registries.push_back(getFlagRegistry()); + return registries; } Value * makeFlakeRegistryValue(EvalState & state) { auto v = state.allocValue(); - auto registry = state.getFlakeRegistry(); + auto registries = state.getFlakeRegistries(); - state.mkAttrs(*v, registry.entries.size()); + int size = 0; + for (auto registry : registries) + size += registry->entries.size(); + state.mkAttrs(*v, size); - for (auto & entry : registry.entries) { - auto vEntry = state.allocAttr(*v, entry.first); - state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); - vEntry->attrs->sort(); + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + auto vEntry = state.allocAttr(*v, entry.first); + state.mkAttrs(*vEntry, 2); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); + vEntry->attrs->sort(); + } } v->attrs->sort(); @@ -89,7 +118,7 @@ Value * makeFlakeRegistryValue(EvalState & state) } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, - std::vector registries) + std::vector> registries) { if (auto refData = std::get_if(&flakeRef.data)) { for (auto registry : registries) { @@ -117,13 +146,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) FlakeRef directFlakeRef = FlakeRef(flakeRef); if (!flakeRef.isDirect()) { - std::vector registries; - // 'pureEval' is a setting which cannot be changed in `nix flake`, - // but without flagging it off, we can't use any FlakeIds. - // if (!evalSettings.pureEval) { - registries.push_back(&state.getFlakeRegistry()); - // } - directFlakeRef = lookupFlake(state, flakeRef, registries); + directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); } assert(directFlakeRef.isDirect()); // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes.. @@ -246,11 +269,8 @@ static std::tuple> resolveFlake(EvalState & st std::optional topFlakeId; /// FIXME: ambiguous todo.push({topRef, true}); - std::vector registries; - FlakeRegistry localRegistry; - registries.push_back(&localRegistry); - if (!evalSettings.pureEval) - registries.push_back(&state.getFlakeRegistry()); + std::vector> registries = state.getFlakeRegistries(); + std::shared_ptr localRegistry = registries.at(2); while (!todo.empty()) { auto [flakeRef, toplevel] = todo.front(); @@ -259,6 +279,7 @@ static std::tuple> resolveFlake(EvalState & st if (auto refData = std::get_if(&flakeRef.data)) { if (done.count(refData->id)) continue; // optimization flakeRef = lookupFlake(state, flakeRef, registries); + // This is why we need the `registries`. } if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) @@ -273,10 +294,13 @@ static std::tuple> resolveFlake(EvalState & st for (auto & require : flake.requires) todo.push({require, false}); + // The following piece of code basically adds the FlakeRefs from + // the lockfiles of dependencies to the localRegistry. This is used + // to resolve future `FlakeId`s, in `lookupFlake` a bit above this. if (flake.lockFile) for (auto & entry : flake.lockFile->entries) { - if (localRegistry.entries.count(entry.first)) continue; - localRegistry.entries.emplace(entry.first, entry.second); + if (localRegistry->entries.count(entry.first)) continue; + localRegistry->entries.emplace(entry.first, entry.second); } done.emplace(flake.id, std::move(flake)); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 4e49becc7..53cea1cc2 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -25,7 +25,7 @@ Value * makeFlakeRegistryValue(EvalState & state); Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); -std::unique_ptr readRegistry(const Path &); +std::shared_ptr readRegistry(const Path &); void writeRegistry(FlakeRegistry, Path); @@ -36,7 +36,7 @@ struct Flake std::string description; Path path; std::vector requires; - std::unique_ptr lockFile; + std::shared_ptr lockFile; Value * vProvides; // FIXME: gc // commit hash // date diff --git a/src/nix/flake.cc b/src/nix/flake.cc index fda903944..470dfdc08 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -24,12 +24,14 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs { auto evalState = std::make_shared(searchPath, store); - auto registry = evalState->getFlakeRegistry(); + auto registries = evalState->getFlakeRegistries(); stopProgressBar(); - for (auto & entry : registry.entries) { - std::cout << entry.first << " " << entry.second.ref.to_string() << "\n"; + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + std::cout << entry.first << " " << entry.second.ref.to_string() << "\n"; + } } } }; From fcd766097636943ff11b84747d11a4e52d3d8e38 Mon Sep 17 00:00:00 2001 From: xbreak Date: Sun, 24 Mar 2019 09:39:48 +0000 Subject: [PATCH 0075/2548] repl: Restore CTRL-C behaviour Install signal handler during `readline` to handle SIGINT to abort partially typed expressions. --- src/nix/repl.cc | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 227affc60..d8f812149 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -192,6 +192,14 @@ static int listPossibleCallback(char *s, char ***avp) { return ac; } +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} void NixRepl::mainLoop(const std::vector & files) { @@ -251,8 +259,40 @@ void NixRepl::mainLoop(const std::vector & files) bool NixRepl::getLine(string & input, const std::string &prompt) { + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + if (!s) return false; input += s; From 514b3c7f8345cfcbbe166981214497ed9d93ae18 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 29 Jan 2018 12:36:59 +0000 Subject: [PATCH 0076/2548] Add isPath primop this is added for completeness' sake since all the other possible `builtins.typeOf` results have a corresponding `builtins.is` --- doc/manual/expressions/builtins.xml | 8 ++++++++ src/libexpr/primops.cc | 7 +++++++ tests/lang/eval-okay-types.nix | 2 ++ 3 files changed, 17 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index cefc8b78e..0fb5261b3 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -861,6 +861,14 @@ x: x + 456 + builtins.isPath + e + + Return true if + e evaluates to a path, and + false otherwise. + + isNull diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6b0c55e72..39073725e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -2169,6 +2175,7 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); addPrimOp("__addErrorContext", 2, prim_addErrorContext); diff --git a/tests/lang/eval-okay-types.nix b/tests/lang/eval-okay-types.nix index a34775f5e..9b58be5d1 100644 --- a/tests/lang/eval-okay-types.nix +++ b/tests/lang/eval-okay-types.nix @@ -20,6 +20,8 @@ with builtins; (isFloat (1 - 2.0)) (isBool (true && false)) (isBool null) + (isPath /nix/store) + (isPath ./.) (isAttrs { x = 123; }) (isAttrs null) (typeOf (3 * 4)) From 0bebca402afa5f334c6d268d937097efbe1762d5 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 09:55:30 -0500 Subject: [PATCH 0077/2548] version -> .version, avoid conflict with C++20 --- version => .version | 0 release.nix | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename version => .version (100%) diff --git a/version b/.version similarity index 100% rename from version rename to .version diff --git a/release.nix b/release.nix index 5c0027301..55808407e 100644 --- a/release.nix +++ b/release.nix @@ -18,7 +18,7 @@ let releaseTools.sourceTarball { name = "nix-tarball"; - version = builtins.readFile ./version; + version = builtins.readFile ./.version; versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"; src = nix; inherit officialRelease; From 42e2d5e7b7214772a0afa57e87de08715994dc29 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:00:00 -0500 Subject: [PATCH 0078/2548] store-api.hh: add missing include for unordered_map --- src/libstore/store-api.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 40887b6aa..7a1b31d0f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include From 1f9c8cd68b9cd06a7c604ee7a86a83eef676273c Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:21:22 -0500 Subject: [PATCH 0079/2548] configure.ac: update for new version file location --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4eea008b0..f5b1614f1 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./version)$VERSION_SUFFIX"])) +AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) From 63e7fc509628874bf60d9e436af387bfb1bc00a5 Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Thu, 14 Mar 2019 10:28:10 -0500 Subject: [PATCH 0080/2548] perl/configure.ac: fix for new version location too --- perl/configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/configure.ac b/perl/configure.ac index 9f49db4d2..966700695 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../version)$VERSION_SUFFIX"])) +AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) From 75ec68f93ac98956552c06a7fdf4dc8da6841a65 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 25 Mar 2019 20:55:28 -0400 Subject: [PATCH 0081/2548] manual: mention the "channel:" shorthand for NIX_PATH Bumped to 15.09 because older channels, when downloaded from nixos.org, require firefox to be accessed via `pkgs.firefox` --- doc/manual/command-ref/env-common.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 361d3e2b0..c532ffdde 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -52,10 +52,15 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos NIX_PATH to -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz +nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz tells Nix to download the latest revision in the Nixpkgs/NixOS - 14.12 channel. + 15.09 channel. + + A following shorthand can be used to refer to the official channels: + + nixpkgs=channel:nixos-15.09 + The search path can be extended using the option, which takes precedence over From f9c7176a87ccc71b719689cb28f8bc1bfb5354e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Mar 2019 12:48:57 +0100 Subject: [PATCH 0082/2548] nix flake add: Handle ~/.config/nix not existing Fixes $ nix flake add fnord github:edolstra/fnord error: opening file '/home/eelco/.config/nix/registry.json': No such file or directory --- src/libexpr/primops/flake.cc | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index b74e0b4b7..00eeba632 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -23,16 +23,20 @@ std::unique_ptr readRegistry(const Path & path) { auto registry = std::make_unique(); - auto json = nlohmann::json::parse(readFile(path)); + try { + auto json = nlohmann::json::parse(readFile(path)); - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - registry->entries.emplace(i.key(), entry); + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; + registry->entries.emplace(i.key(), entry); + } + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; } return registry; @@ -47,6 +51,7 @@ void writeRegistry(FlakeRegistry registry, Path path) for (auto elem : registry.entries) { json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; } + createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } From be7fd6359559717b83833d96d4b6dc38ceb83092 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 26 Mar 2019 14:25:43 +0100 Subject: [PATCH 0083/2548] Remove debug line --- src/nix/build.cc | 2 +- src/nix/installables.cc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 5ab22e26c..da7c7f614 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,7 +78,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - if(gitRepo) + if (gitRepo) updateLockFile(*evalState, *gitRepo); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 21e9e73b8..0453c72c2 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -234,7 +234,6 @@ Buildables build(ref store, RealiseMode mode, PathSet pathsToBuild; for (auto & i : installables) { - std::cout << i->what() << std::endl; for (auto & b : i->toBuildables()) { if (b.drvPath != "") { StringSet outputNames; From d854e7dfd6d512c8ed687b61d7aae3358eda71cd Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Tue, 26 Mar 2019 21:08:22 -0400 Subject: [PATCH 0084/2548] install-multi-user: Detect and fail lack of systemd separately Otherwise, the user is shown: ``` Sorry, I don't know what to do on Linux ``` Which is... not exactly right. --- scripts/install-multi-user.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 4b65783a2..80d1c73fb 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -753,9 +753,13 @@ main() { if [ "$(uname -s)" = "Darwin" ]; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi else failure "Sorry, I don't know what to do on $(uname)" fi From dbe4c043d75f90f30932d326d2c80fd030b1f06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 27 Mar 2019 16:18:53 +0100 Subject: [PATCH 0085/2548] install-multi-user: reduce max-jobs from 32 to 1 Having max-jobs = 32 ($NIX_USER_COUNT is hardcoded to that value) may severely overload the machine. The nix.conf(5) manual page says max-jobs defaults to 1, so let's use that value. NOTE: Both max-jobs and cores are now being set to their default value, so they can be removed alltogether. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 4b65783a2..6a76761cc 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -742,7 +742,7 @@ place_nix_configuration() { cat < "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME -max-jobs = $NIX_USER_COUNT +max-jobs = 1 cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ From 07d9981f34c8423e74a8dc1b4f978fe58b421aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 27 Mar 2019 16:24:58 +0100 Subject: [PATCH 0086/2548] install-multi-user: remove unneeded settings from nix.conf Hardcoding the "max-jobs" and "cores" settings in nix.conf at install time, to the same value as Nix' built-in default, makes little sense to me. --- scripts/install-multi-user.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 6a76761cc..1e1db21c3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -741,9 +741,6 @@ setup_default_profile() { place_nix_configuration() { cat < "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME - -max-jobs = 1 -cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf From e489f5cabf95aaaaf0e67bb52d27dcd13a6cb1c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 21:07:04 +0100 Subject: [PATCH 0087/2548] Update eval-okay-types.exp to match #1828 --- tests/lang/eval-okay-types.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-types.exp b/tests/lang/eval-okay-types.exp index 9a8ea0bcb..92a153299 100644 --- a/tests/lang/eval-okay-types.exp +++ b/tests/lang/eval-okay-types.exp @@ -1 +1 @@ -[ true false true false true false true false true true true true true true true true true true true false true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] +[ true false true false true false true false true true true true true true true true true true true false true true true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] From f32fbf952d8eb15963da65a162da169de72f36fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Mar 2019 21:09:31 +0100 Subject: [PATCH 0088/2548] Fix Bison 3.3 warning --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index cbd576d7d..787039307 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,7 +1,7 @@ %glr-parser %pure-parser %locations -%error-verbose +%define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } From caa76c369a6d77eb3ccaa14bd8144581c8cd50a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sun, 31 Mar 2019 03:47:43 +0200 Subject: [PATCH 0089/2548] docs: Mention `--max-jobs 0` to build remotely only --- doc/manual/advanced-topics/distributed-builds.xml | 3 +++ doc/manual/command-ref/opt-common.xml | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/advanced-topics/distributed-builds.xml b/doc/manual/advanced-topics/distributed-builds.xml index ce2e077ed..9ac4a92cd 100644 --- a/doc/manual/advanced-topics/distributed-builds.xml +++ b/doc/manual/advanced-topics/distributed-builds.xml @@ -184,4 +184,7 @@ to be included. (This is the default.) the option builders-use-substitutes in your local nix.conf. +To build only on remote builders and disable building on the local machine, +you can use the option . + diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 4c572e129..b8a2f260e 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -107,14 +107,22 @@ / number - Sets the maximum number of build jobs that Nix will + + + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. Specify auto to use the number of CPUs in the system. The default is specified by the max-jobs configuration setting, which itself defaults to 1. A higher value is useful on SMP systems or to - exploit I/O latency. + exploit I/O latency. + + Setting it to 0 disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders. + + From 2f59b302517dff622743eb081884deacc906b3d1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Apr 2019 21:09:25 +0200 Subject: [PATCH 0090/2548] Use Nixpkgs 19.03 --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index 55808407e..ab13451ff 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; } +, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; } , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index 817684b76..73e75fb29 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-18.09"; }) {}; +with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {}; with import ./release-common.nix { inherit pkgs; }; From 2bc6304793e14e44d6d24c1fb35666352c7106f2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Apr 2019 21:09:35 +0200 Subject: [PATCH 0091/2548] getMachines(): Cache result --- src/libstore/machines.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index edd03d147..f848582da 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines) Machines getMachines() { - Machines machines; - - parseMachines(settings.builders, machines); - + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); return machines; } From 41ba5135e0a2d9940c4e8cbedff44eb1a47af782 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 1 Apr 2019 21:09:49 +0200 Subject: [PATCH 0092/2548] primeCache(): Barf if builds are needed but not allowed Fixes #2753. --- src/libstore/build.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6b88b1307..53a0c743b 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -12,6 +12,7 @@ #include "json.hh" #include "nar-info.hh" #include "parsed-derivations.hh" +#include "machines.hh" #include #include @@ -4411,6 +4412,11 @@ static void primeCache(Store & store, const PathSet & paths) PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); } From 6b0ca8e803710342af70e257935724c5ad84ca04 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 14:20:58 +0200 Subject: [PATCH 0093/2548] findAlongAttrPath(): Throw AttrPathNotFound --- src/libexpr/attr-path.cc | 4 ++-- src/libexpr/attr-path.hh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index b0f80db32..832235cfd 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -70,7 +70,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; } @@ -82,7 +82,7 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, % attrPath % showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 46a341950..1eae64625 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,6 +7,8 @@ namespace nix { +MakeError(AttrPathNotFound, Error); + Value * findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); From 154244adc6c9831e00a41bf7799a2d29c6a3a3b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 14:21:13 +0200 Subject: [PATCH 0094/2548] nix: New installables syntax The general syntax for an installable is now :. The attrpath is relative to the flake's 'provides.packages' or 'provides' if the former doesn't yield a result. E.g. $ nix build nixpkgs:hello is equivalent to $ nix build nixpkgs:packages.hello Also, ':' can be omitted, in which case it defaults to 'nixpkgs', e.g. $ nix build hello --- src/libexpr/primops/flake.cc | 55 +++++++++------------------- src/libexpr/primops/flake.hh | 2 +- src/nix/installables.cc | 71 +++++++++++++++++++++++++++--------- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index a8d46825f..dedd2f737 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -57,8 +57,9 @@ Path getUserRegistryPath() std::shared_ptr getGlobalRegistry() { - // TODO: Make a global registry, and return it here. - return std::make_shared(); + // FIXME: get from nixos.org. + Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; + return readRegistry(registryFile); } std::shared_ptr getUserRegistry() @@ -66,33 +67,17 @@ std::shared_ptr getUserRegistry() return readRegistry(getUserRegistryPath()); } -// Project-specific registry saved in flake-registry.json. -std::shared_ptr getLocalRegistry() -{ - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); -} - std::shared_ptr getFlagRegistry() { return std::make_shared(); // TODO: Implement this once the right flags are implemented. } -// This always returns a vector with globalReg, userReg, localReg, flakeReg. -// If one of them doesn't exist, the registry is left empty but does exist. const std::vector> EvalState::getFlakeRegistries() { std::vector> registries; - if (!evalSettings.pureEval) { - registries.push_back(std::make_shared()); // global - registries.push_back(std::make_shared()); // user - registries.push_back(std::make_shared()); // local - } else { - registries.push_back(getGlobalRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getLocalRegistry()); - } + registries.push_back(getGlobalRegistry()); + registries.push_back(getUserRegistry()); registries.push_back(getFlagRegistry()); return registries; } @@ -149,8 +134,7 @@ struct FlakeSourceInfo static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) { FlakeRef directFlakeRef = FlakeRef(flakeRef); - if (!flakeRef.isDirect()) - { + if (!flakeRef.isDirect()) { directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); } assert(directFlakeRef.isDirect()); @@ -274,8 +258,8 @@ static std::tuple> resolveFlake(EvalState & st std::optional topFlakeId; /// FIXME: ambiguous todo.push({topRef, true}); - std::vector> registries = state.getFlakeRegistries(); - std::shared_ptr localRegistry = registries.at(2); + auto registries = state.getFlakeRegistries(); + //std::shared_ptr localRegistry = registries.at(2); while (!todo.empty()) { auto [flakeRef, toplevel] = todo.front(); @@ -283,12 +267,15 @@ static std::tuple> resolveFlake(EvalState & st if (auto refData = std::get_if(&flakeRef.data)) { if (done.count(refData->id)) continue; // optimization - flakeRef = lookupFlake(state, flakeRef, registries); + flakeRef = lookupFlake(state, flakeRef, + !evalSettings.pureEval || (toplevel && impureTopRef) ? registries : std::vector>()); // This is why we need the `registries`. } +#if 0 if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --no-pure-eval to disable", flakeRef.to_string()); +#endif auto flake = getFlake(state, flakeRef); @@ -299,6 +286,7 @@ static std::tuple> resolveFlake(EvalState & st for (auto & require : flake.requires) todo.push({require, false}); +#if 0 // The following piece of code basically adds the FlakeRefs from // the lockfiles of dependencies to the localRegistry. This is used // to resolve future `FlakeId`s, in `lookupFlake` a bit above this. @@ -307,6 +295,7 @@ static std::tuple> resolveFlake(EvalState & st if (localRegistry->entries.count(entry.first)) continue; localRegistry->entries.emplace(entry.first, entry.second); } +#endif done.emplace(flake.id, std::move(flake)); } @@ -344,19 +333,9 @@ void updateLockFile(EvalState & state, std::string path) } } -Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v) +Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) { - // FIXME: temporary hack to make the default installation source - // work. - bool impure = false; - if (hasPrefix(flakeUri, "impure:")) { - flakeUri = std::string(flakeUri, 7); - impure = true; - } - - auto flakeRef = FlakeRef(flakeUri); - - auto [topFlakeId, flakes] = resolveFlake(state, flakeUri, impure); + auto [topFlakeId, flakes] = resolveFlake(state, flakeRef, impureTopRef); // FIXME: we should call each flake with only its dependencies // (rather than the closure of the top-level flake). @@ -387,7 +366,7 @@ Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v) static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), v); + makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 53cea1cc2..df8cf9efb 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -23,7 +23,7 @@ Path getUserRegistryPath(); Value * makeFlakeRegistryValue(EvalState & state); -Value * makeFlakeValue(EvalState & state, std::string flakeUri, Value & v); +Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v); std::shared_ptr readRegistry(const Path &); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0453c72c2..5e4ea6054 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -21,13 +21,6 @@ SourceExprCommand::SourceExprCommand() .label("file") .description("evaluate FILE rather than use the default installation source") .dest(&file); - - mkFlag() - .shortName('F') - .longName("flake") - .label("flake") - .description("evaluate FLAKE rather than use the default installation source") - .dest(&flakeUri); } Value * SourceExprCommand::getSourceExpr(EvalState & state) @@ -36,17 +29,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) vSourceExpr = state.allocValue(); - if (file && flakeUri) - throw Error("cannot use both --file and --flake"); - if (file) state.evalFile(lookupFileArg(state, *file), *vSourceExpr); - else if (flakeUri) { - // FIXME: handle flakeUri being a relative path - auto vTemp = state.allocValue(); - auto vFlake = *makeFlakeValue(state, "impure:" + *flakeUri, *vTemp); - *vSourceExpr = *((*vFlake.attrs->get(state.symbols.create("provides")))->value); - } else { + else { // FIXME: remove "impure" hack, call some non-user-accessible // variant of getFlake instead. auto fun = state.parseExprFromString( @@ -176,6 +161,43 @@ struct InstallableAttrPath : InstallableValue } }; +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + std::string attrPath; + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, const std::string & attrPath) + : InstallableValue(cmd), flakeRef(flakeRef), attrPath(attrPath) + { } + + std::string what() override { return flakeRef.to_string() + ":" + attrPath; } + + Value * toValue(EvalState & state) override + { + auto vTemp = state.allocValue(); + auto vFlake = *makeFlakeValue(state, flakeRef, true, *vTemp); + + auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; + + state.forceValue(*vProvides); + + auto emptyArgs = state.allocBindings(0); + + if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + try { + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); + state.forceValue(*v); + return v; + } +}; + // FIXME: extend std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); @@ -196,19 +218,34 @@ static std::vector> parseInstallables( if (s.compare(0, 1, "(") == 0) result.push_back(std::make_shared(cmd, s)); - else if (s.find("/") != std::string::npos) { + /* + else if (s.find('/') != std::string::npos) { auto path = store->toStorePath(store->followLinksToStore(s)); if (store->isStorePath(path)) result.push_back(std::make_shared(path)); } + */ + else { + auto colon = s.rfind(':'); + if (colon != std::string::npos) { + auto flakeRef = std::string(s, 0, colon); + auto attrPath = std::string(s, colon + 1); + result.push_back(std::make_shared(cmd, FlakeRef(flakeRef), attrPath)); + } else { + result.push_back(std::make_shared(cmd, FlakeRef("nixpkgs"), s)); + } + } + + /* else if (s == "" || std::regex_match(s, attrPathRegex)) result.push_back(std::make_shared(cmd, s)); else throw UsageError("don't know what to do with argument '%s'", s); + */ } return result; From 101d964a59d5c9098845d3109ea1eba99b5f31df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 16:11:17 +0200 Subject: [PATCH 0095/2548] nix: Make -f work for compatibility --- src/nix/command.hh | 23 ++------ src/nix/installables.cc | 121 +++++++++++++++++----------------------- src/nix/search.cc | 4 +- src/nix/why-depends.cc | 4 +- 4 files changed, 61 insertions(+), 91 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index ffe64ccb7..83959bf9a 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -74,23 +74,20 @@ struct Installable struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { std::optional file; - std::optional flakeUri; SourceExprCommand(); - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); - ref getEvalState(); + std::vector> parseInstallables( + ref store, std::vector ss); + + std::shared_ptr parseInstallable( + ref store, const std::string & installable); + private: std::shared_ptr evalState; - - Value * vSourceExpr = 0; }; enum RealiseMode { Build, NoBuild, DryRun }; @@ -108,8 +105,6 @@ struct InstallablesCommand : virtual Args, SourceExprCommand void prepare() override; - virtual bool useDefaultInstallables() { return true; } - private: std::vector _installables; @@ -148,8 +143,6 @@ public: virtual void run(ref store, Paths storePaths) = 0; void run(ref store) override; - - bool useDefaultInstallables() override { return !all; } }; /* A command that operates on exactly one store path. */ @@ -174,10 +167,6 @@ struct RegisterCommand } }; -std::shared_ptr parseInstallable( - SourceExprCommand & cmd, ref store, const std::string & installable, - bool useDefaultInstallables); - Buildables build(ref store, RealiseMode mode, std::vector> installables); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 5e4ea6054..bcb22349a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -19,33 +19,10 @@ SourceExprCommand::SourceExprCommand() .shortName('f') .longName("file") .label("file") - .description("evaluate FILE rather than use the default installation source") + .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); } -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return vSourceExpr; - - vSourceExpr = state.allocValue(); - - if (file) - state.evalFile(lookupFileArg(state, *file), *vSourceExpr); - else { - // FIXME: remove "impure" hack, call some non-user-accessible - // variant of getFlake instead. - auto fun = state.parseExprFromString( - "builtins.mapAttrs (flakeName: flakeInfo:" - " (getFlake (\"impure:\" + flakeInfo.uri)).${flakeName}.provides.packages or {})", "/"); - auto vFun = state.allocValue(); - state.eval(fun, *vFun); - auto vRegistry = makeFlakeRegistryValue(state); - mkApp(*vSourceExpr, *vFun, *vRegistry); - } - - return vSourceExpr; -} - ref SourceExprCommand::getEvalState() { if (!evalState) @@ -140,24 +117,20 @@ struct InstallableExpr : InstallableValue struct InstallableAttrPath : InstallableValue { + Value * v; std::string attrPath; - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) + InstallableAttrPath(SourceExprCommand & cmd, Value * v, const std::string & attrPath) + : InstallableValue(cmd), v(v), attrPath(attrPath) { } std::string what() override { return attrPath; } Value * toValue(EvalState & state) override { - auto source = cmd.getSourceExpr(state); - - Bindings & autoArgs = *cmd.getAutoArgs(state); - - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); - state.forceValue(*v); - - return v; + auto vRes = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), *v); + state.forceValue(*vRes); + return vRes; } }; @@ -202,60 +175,66 @@ struct InstallableFlake : InstallableValue std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); -static std::vector> parseInstallables( - SourceExprCommand & cmd, ref store, std::vector ss, bool useDefaultInstallables) +std::vector> SourceExprCommand::parseInstallables( + ref store, std::vector ss) { std::vector> result; - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } + if (file) { + // FIXME: backward compatibility hack + evalSettings.pureEval = false; - for (auto & s : ss) { + auto state = getEvalState(); + auto vFile = state->allocValue(); + state->evalFile(lookupFileArg(*state, *file), *vFile); - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared(cmd, s)); + if (ss.empty()) + ss = {""}; - /* - else if (s.find('/') != std::string::npos) { + for (auto & s : ss) + result.push_back(std::make_shared(*this, vFile, s)); - auto path = store->toStorePath(store->followLinksToStore(s)); + } else { - if (store->isStorePath(path)) - result.push_back(std::make_shared(path)); - } - */ + for (auto & s : ss) { - else { - auto colon = s.rfind(':'); - if (colon != std::string::npos) { + size_t colon; + + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared(*this, s)); + + else if ((colon = s.rfind(':')) != std::string::npos) { auto flakeRef = std::string(s, 0, colon); auto attrPath = std::string(s, colon + 1); - result.push_back(std::make_shared(cmd, FlakeRef(flakeRef), attrPath)); - } else { - result.push_back(std::make_shared(cmd, FlakeRef("nixpkgs"), s)); + result.push_back(std::make_shared(*this, FlakeRef(flakeRef), attrPath)); } + + else if (s.find('/') != std::string::npos) { + auto path = store->toStorePath(store->followLinksToStore(s)); + result.push_back(std::make_shared(path)); + } + + else { + result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), s)); + } + + /* + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared(cmd, s)); + + else + throw UsageError("don't know what to do with argument '%s'", s); + */ } - - /* - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared(cmd, s)); - - else - throw UsageError("don't know what to do with argument '%s'", s); - */ } return result; } -std::shared_ptr parseInstallable( - SourceExprCommand & cmd, ref store, const std::string & installable, - bool useDefaultInstallables) +std::shared_ptr SourceExprCommand::parseInstallable( + ref store, const std::string & installable) { - auto installables = parseInstallables(cmd, store, {installable}, false); + auto installables = parseInstallables(store, {installable}); assert(installables.size() == 1); return installables.front(); } @@ -342,12 +321,12 @@ PathSet toDerivations(ref store, void InstallablesCommand::prepare() { - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); + installables = parseInstallables(getStore(), _installables); } void InstallableCommand::prepare() { - installable = parseInstallable(*this, getStore(), _installable, false); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/nix/search.cc b/src/nix/search.cc index e086de226..55f8d106a 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -257,7 +257,9 @@ struct CmdSearch : SourceExprCommand, MixJSON auto cache = writeCache ? std::make_unique(jsonCacheFile, false) : nullptr; - doExpr(getSourceExpr(*state), "", true, cache.get()); + // FIXME + throw Error("NOT IMPLEMENTED"); + //doExpr(getSourceExpr(*state), "", true, cache.get()); } catch (std::exception &) { /* Fun fact: catching std::ios::failure does not work diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 325a2be0a..32ba5a1ad 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -74,9 +74,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref store) override { - auto package = parseInstallable(*this, store, _package, false); + auto package = parseInstallable(store, _package); auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependency = parseInstallable(store, _dependency); auto dependencyPath = toStorePath(store, NoBuild, dependency); auto dependencyPathHash = storePathToHash(dependencyPath); From d2875f678270b4c241055765ec65d9ddb66bd60f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 16:11:36 +0200 Subject: [PATCH 0096/2548] Fix tests --- tests/fetchGit.sh | 42 ++++++++++++++++++++--------------------- tests/fetchMercurial.sh | 28 +++++++++++++-------------- tests/pure-eval.sh | 16 ++++++++-------- tests/restricted.sh | 16 ++++++++-------- tests/search.sh | 2 ++ 5 files changed, 53 insertions(+), 51 deletions(-) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 4c46bdf04..301bf3022 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -26,30 +26,30 @@ git -C $repo commit -m 'Bla2' -a rev2=$(git -C $repo rev-parse HEAD) # Fetch the default branch. -path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] -(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") +[[ $(nix eval --no-pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] +(! nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") # Fetch using an explicit revision hash. path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] # In pure eval mode, fetchGit with a revision should succeed. -[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] +[[ $(nix eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] -[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --no-pure-eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) +(! nix eval --no-pure-eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -61,7 +61,7 @@ path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -72,17 +72,17 @@ echo bar > $repo/dir2/bar git -C $repo add dir1/foo git -C $repo rm hello -path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit ref or rev. -path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path3=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") @@ -91,7 +91,7 @@ path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).ou # Committing should not affect the store path. git -C $repo commit -m 'Bla3' -a -path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") +path4=$(nix eval --no-pure-eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] # tarball-ttl should be ignored if we specify a rev @@ -102,29 +102,29 @@ rev3=$(git -C $repo rev-parse HEAD) nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null # Update 'path' to reflect latest master -path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") # Check behavior when non-master branch is used git -C $repo checkout $rev2 -b dev echo dev > $repo/hello # File URI uses 'master' unless specified otherwise -path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $path = $path2 ]] # Using local path with branch other than 'master' should work when clean or dirty -path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path3=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a -path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +path4=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") [[ $(cat $path4/hello) = dev ]] [[ $path3 = $path4 ]] # Confirm same as 'dev' branch -path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -134,8 +134,8 @@ rm -rf $TEST_HOME/.cache/nix/gitv2 # Try again, but without 'git' on PATH NIX=$(command -v nix) # This should fail -(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) +(! PATH= $NIX eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) # Try again, with 'git' available. This should work. -path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index 4088dbd39..d0735a381 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -26,31 +26,31 @@ hg commit --cwd $repo -m 'Bla2' rev2=$(hg log --cwd $repo -r tip --template '{node}') # Fetch the default branch. -path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] -(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") +[[ $(nix eval --no-pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] +(! nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") # Fetch using an explicit revision hash. -path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] # In pure eval mode, fetchGit with a revision should succeed. -[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] +[[ $(nix eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] -[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] -[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval --no-pure-eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") +(! nix eval --no-pure-eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -62,7 +62,7 @@ path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file:// mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -73,14 +73,14 @@ echo bar > $repo/dir2/bar hg add --cwd $repo dir1/foo hg rm --cwd $repo hello -path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.hg ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit rev. path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") @@ -89,5 +89,5 @@ path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\ # Committing should not affect the store path. hg commit --cwd $repo -m 'Bla3' -path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +path4=$(nix eval --no-pure-eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $path2 = $path4 ]] diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 49c856448..307942940 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -2,17 +2,17 @@ source common.sh clearStore -nix eval --pure-eval '(assert 1 + 2 == 3; true)' +nix eval '(assert 1 + 2 == 3; true)' -[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] +[[ $(nix eval --no-pure-eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] -(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)') +(! nix eval '(builtins.readFile ./pure-eval.sh)') -(! nix eval --pure-eval '(builtins.currentTime)') -(! nix eval --pure-eval '(builtins.currentSystem)') +(! nix eval '(builtins.currentTime)') +(! nix eval '(builtins.currentSystem)') (! nix-instantiate --pure-eval ./simple.nix) -[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] -(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") -nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" +[[ $(nix eval --no-pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] +(! nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") +nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" diff --git a/tests/restricted.sh b/tests/restricted.sh index e02becc60..68913cd36 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,18 +17,18 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. -p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)") +p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval) +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval) -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") -nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" +nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" -(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) -(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) -(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval) +(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --no-pure-eval --restrict-eval) +(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --no-pure-eval --restrict-eval) +(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --no-pure-eval --restrict-eval) ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix [[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] @@ -37,7 +37,7 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . -[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] +[[ $(nix eval --raw --no-pure-eval --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" diff --git a/tests/search.sh b/tests/search.sh index 14da3127b..6c4d791c1 100644 --- a/tests/search.sh +++ b/tests/search.sh @@ -3,6 +3,8 @@ source common.sh clearStore clearCache +exit 0 # FIXME + # No packages (( $(NIX_PATH= nix search -u|wc -l) == 0 )) From 4023ae4cdf146b2ee491c12ec64e8605984bf49a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 16:22:04 +0200 Subject: [PATCH 0097/2548] nix: Support nixpkgs. for compatibility --- src/nix/installables.cc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index bcb22349a..6d3969e95 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -203,6 +203,12 @@ std::vector> SourceExprCommand::parseInstallables( if (s.compare(0, 1, "(") == 0) result.push_back(std::make_shared(*this, s)); + else if (hasPrefix(s, "nixpkgs.")) { + bool static warned; + warnOnce(warned, "the syntax 'nixpkgs.' is deprecated; use 'nixpkgs:' instead"); + result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), std::string(s, 8))); + } + else if ((colon = s.rfind(':')) != std::string::npos) { auto flakeRef = std::string(s, 0, colon); auto attrPath = std::string(s, colon + 1); @@ -214,17 +220,8 @@ std::vector> SourceExprCommand::parseInstallables( result.push_back(std::make_shared(path)); } - else { - result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), s)); - } - - /* - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared(cmd, s)); - else - throw UsageError("don't know what to do with argument '%s'", s); - */ + result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), s)); } } From a9ceeeb4b0caf6891c8cd8fcbe744d3d567c1d8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 17:28:05 +0200 Subject: [PATCH 0098/2548] Add a flake.nix --- flake.nix | 18 ++++++++++++++++++ release.nix | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 flake.nix diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..b119f0324 --- /dev/null +++ b/flake.nix @@ -0,0 +1,18 @@ +{ + name = "nix"; + + description = "The purely functional package manager"; + + requires = [ flake:nixpkgs ]; + + provides = flakes: rec { + + hydraJobs = import ./release.nix { + nix = flakes.nix; # => flakes.self? + nixpkgs = flakes.nixpkgs; + }; + + packages.nix = hydraJobs.build.x86_64-linux; + + }; +} diff --git a/release.nix b/release.nix index ab13451ff..f52120474 100644 --- a/release.nix +++ b/release.nix @@ -19,7 +19,7 @@ let releaseTools.sourceTarball { name = "nix-tarball"; version = builtins.readFile ./.version; - versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"; + versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; src = nix; inherit officialRelease; From 6a4c7fb9759dbbf5ddaf0ebd00921d0f8045f355 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 22:46:25 +0200 Subject: [PATCH 0099/2548] Add path flakeref variant Unlike file://, this allows the path to be a dirty Git tree, so nix build /path/to/flake:attr is a convenient way to test building a local flake. --- src/libexpr/primops/fetchGit.cc | 4 ++-- src/libexpr/primops/fetchGit.hh | 2 +- src/libexpr/primops/flake.cc | 27 ++++++++++++++++++++++++++- src/libexpr/primops/flake.hh | 1 + src/libexpr/primops/flakeref.cc | 13 +++++++++++++ src/libexpr/primops/flakeref.hh | 7 ++++++- 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index bbf13c87b..391308224 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -170,7 +170,7 @@ GitInfo exportGit(ref store, const std::string & uri, json["uri"] = uri; json["name"] = name; json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; + json["revCount"] = *gitInfo.revCount; writeFile(storeLink, json.dump()); @@ -224,7 +224,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount.value_or(0)); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index d7a0e165a..60c439426 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -11,7 +11,7 @@ struct GitInfo Path storePath; std::string rev; std::string shortRev; - uint64_t revCount = 0; + std::optional revCount; }; GitInfo exportGit(ref store, const std::string & uri, diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index dedd2f737..f068569a6 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -129,6 +129,7 @@ struct FlakeSourceInfo { Path storePath; std::optional rev; + std::optional revCount; }; static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) @@ -178,6 +179,18 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) FlakeSourceInfo info; info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); + info.revCount = gitInfo.revCount; + return info; + } + + else if (auto refData = std::get_if(&directFlakeRef.data)) { + if (!pathExists(refData->path + "/.git")) + throw Error("flake '%s' does not reference a Git repository", refData->path); + auto gitInfo = exportGit(state.store, refData->path, {}, "", "source"); + FlakeSourceInfo info; + info.storePath = gitInfo.storePath; + info.rev = Hash(gitInfo.rev, htSHA1); + info.revCount = gitInfo.revCount; return info; } @@ -206,6 +219,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } Flake flake(newFlakeRef); + flake.path = flakePath; + flake.revCount = sourceInfo.revCount; Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -349,10 +364,20 @@ Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impure for (auto & flake : flakes) { auto vFlake = state.allocAttr(*vResult, flake.second.id); if (topFlakeId == flake.second.id) vTop = vFlake; - state.mkAttrs(*vFlake, 2); + + state.mkAttrs(*vFlake, 4); + mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); + + state.store->assertStorePath(flake.second.path); + mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.second.path, {flake.second.path}); + + if (flake.second.revCount) + mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.second.revCount); + auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); mkApp(*vProvides, *flake.second.vProvides, *vResult); + vFlake->attrs->sort(); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index df8cf9efb..aea4e8aa2 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -35,6 +35,7 @@ struct Flake FlakeRef ref; std::string description; Path path; + std::optional revCount; std::vector requires; std::shared_ptr lockFile; Value * vProvides; // FIXME: gc diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 8e7c1f8df..5f9a29260 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -106,6 +106,12 @@ FlakeRef::FlakeRef(const std::string & uri) data = d; } + else if (hasPrefix(uri, "/")) { + IsPath d; + d.path = canonPath(uri); + data = d; + } + else throw Error("'%s' is not a valid flake reference", uri); } @@ -135,6 +141,10 @@ std::string FlakeRef::to_string() const (refData->rev ? "&rev=" + refData->rev->to_string(Base16, false) : ""); } + else if (auto refData = std::get_if(&data)) { + return refData->path; + } + else abort(); } @@ -149,6 +159,9 @@ bool FlakeRef::isImmutable() const else if (auto refData = std::get_if(&data)) return (bool) refData->rev; + else if (std::get_if(&data)) + return false; + else abort(); } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index fb365e101..832d7dd03 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -122,9 +122,14 @@ struct FlakeRef std::optional rev; }; + struct IsPath + { + Path path; + }; + // Git, Tarball - std::variant data; + std::variant data; // Parse a flake URI. FlakeRef(const std::string & uri); From ee1254d4f50f5908fa4913253a643d14cb263c45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 23:19:19 +0200 Subject: [PATCH 0100/2548] nix: Add --impure as a shorter alias of --no-pure-eval --- src/libexpr/primops/flake.cc | 2 +- src/nix/installables.cc | 7 +++++++ tests/fetchGit.sh | 38 ++++++++++++++++++------------------ tests/fetchMercurial.sh | 24 +++++++++++------------ tests/pure-eval.sh | 4 ++-- tests/restricted.sh | 16 +++++++-------- 6 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f068569a6..7cfb2038c 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -289,7 +289,7 @@ static std::tuple> resolveFlake(EvalState & st #if 0 if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) - throw Error("mutable flake '%s' is not allowed in pure mode; use --no-pure-eval to disable", flakeRef.to_string()); + throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); #endif auto flake = getFlake(state, flakeRef); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 6d3969e95..631a849cd 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -21,6 +21,13 @@ SourceExprCommand::SourceExprCommand() .label("file") .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); + + mkFlag() + .longName("impure") + .description("allow access to mutable paths and repositories") + .handler([&](std::vector ss) { + evalSettings.pureEval = false; + }); } ref SourceExprCommand::getEvalState() diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 301bf3022..51fd49e9f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -26,11 +26,11 @@ git -C $repo commit -m 'Bla2' -a rev2=$(git -C $repo rev-parse HEAD) # Fetch the default branch. -path=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --impure --raw "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --no-pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] +[[ $(nix eval --impure --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] (! nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") # Fetch using an explicit revision hash. @@ -42,14 +42,14 @@ path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\" # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchGit file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval --no-pure-eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --impure --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --no-pure-eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) +(! nix eval --impure --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -61,7 +61,7 @@ path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchGit $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -72,17 +72,17 @@ echo bar > $repo/dir2/bar git -C $repo add dir1/foo git -C $repo rm hello -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.git ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit ref or rev. -path3=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +path3=$(nix eval --impure --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") [[ $path = $path3 ]] path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") @@ -91,7 +91,7 @@ path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).ou # Committing should not affect the store path. git -C $repo commit -m 'Bla3' -a -path4=$(nix eval --no-pure-eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") +path4=$(nix eval --impure --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] # tarball-ttl should be ignored if we specify a rev @@ -102,29 +102,29 @@ rev3=$(git -C $repo rev-parse HEAD) nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null # Update 'path' to reflect latest master -path=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") +path=$(nix eval --impure --raw "(builtins.fetchGit file://$repo).outPath") # Check behavior when non-master branch is used git -C $repo checkout $rev2 -b dev echo dev > $repo/hello # File URI uses 'master' unless specified otherwise -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchGit file://$repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchGit file://$repo).outPath") [[ $path = $path2 ]] # Using local path with branch other than 'master' should work when clean or dirty -path3=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") +path3=$(nix eval --impure --raw "(builtins.fetchGit $repo).outPath") # (check dirty-tree handling was used) -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] # Committing shouldn't change store path, or switch to using 'master' git -C $repo commit -m 'Bla5' -a -path4=$(nix eval --no-pure-eval --raw "(builtins.fetchGit $repo).outPath") +path4=$(nix eval --impure --raw "(builtins.fetchGit $repo).outPath") [[ $(cat $path4/hello) = dev ]] [[ $path3 = $path4 ]] # Confirm same as 'dev' branch -path5=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --impure --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -134,8 +134,8 @@ rm -rf $TEST_HOME/.cache/nix/gitv2 # Try again, but without 'git' on PATH NIX=$(command -v nix) # This should fail -(! PATH= $NIX eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) +(! PATH= $NIX eval --impure --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) # Try again, with 'git' available. This should work. -path5=$(nix eval --no-pure-eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +path5=$(nix eval --impure --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index d0735a381..a0f792612 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -26,15 +26,15 @@ hg commit --cwd $repo -m 'Bla2' rev2=$(hg log --cwd $repo -r tip --template '{node}') # Fetch the default branch. -path=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path=$(nix eval --impure --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $(cat $path/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. -[[ $(nix eval --no-pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] +[[ $(nix eval --impure --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] (! nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") # Fetch using an explicit revision hash. -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] # In pure eval mode, fetchGit with a revision should succeed. @@ -42,15 +42,15 @@ path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial { url = file://$ # Fetch again. This should be cached. mv $repo ${repo}-tmp -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $path = $path2 ]] -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] -[[ $(nix eval --no-pure-eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval --impure "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --impure --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] # But with TTL 0, it should fail. -(! nix eval --no-pure-eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") +(! nix eval --impure --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") @@ -62,7 +62,7 @@ path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file:// mv ${repo}-tmp $repo # Using a clean working tree should produce the same result. -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchMercurial $repo).outPath") [[ $path = $path2 ]] # Using an unclean tree should yield the tracked but uncommitted changes. @@ -73,14 +73,14 @@ echo bar > $repo/dir2/bar hg add --cwd $repo dir1/foo hg rm --cwd $repo hello -path2=$(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).outPath") +path2=$(nix eval --impure --raw "(builtins.fetchMercurial $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] [ ! -e $path2/dir2/bar ] [ ! -e $path2/.hg ] [[ $(cat $path2/dir1/foo) = foo ]] -[[ $(nix eval --no-pure-eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit rev. path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") @@ -89,5 +89,5 @@ path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\ # Committing should not affect the store path. hg commit --cwd $repo -m 'Bla3' -path4=$(nix eval --no-pure-eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +path4=$(nix eval --impure --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") [[ $path2 = $path4 ]] diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index 307942940..6e2c6962d 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -4,7 +4,7 @@ clearStore nix eval '(assert 1 + 2 == 3; true)' -[[ $(nix eval --no-pure-eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] +[[ $(nix eval --impure '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] (! nix eval '(builtins.readFile ./pure-eval.sh)') @@ -13,6 +13,6 @@ nix eval '(assert 1 + 2 == 3; true)' (! nix-instantiate --pure-eval ./simple.nix) -[[ $(nix eval --no-pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] +[[ $(nix eval --impure "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] (! nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" diff --git a/tests/restricted.sh b/tests/restricted.sh index 68913cd36..e660de127 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -17,18 +17,18 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. -p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)") +p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval) +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --impure --restrict-eval) -(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") -nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --no-pure-eval --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" +nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" -(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --no-pure-eval --restrict-eval) -(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --no-pure-eval --restrict-eval) -(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --no-pure-eval --restrict-eval) +(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --impure --restrict-eval) +(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --impure --restrict-eval) +(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --impure --restrict-eval) ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix [[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] @@ -37,7 +37,7 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . -[[ $(nix eval --raw --no-pure-eval --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] +[[ $(nix eval --raw --impure --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" From 47727252ff4e536dd47b73949033d3349923fbbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 23:36:12 +0200 Subject: [PATCH 0101/2548] Add "nix flake init" command for creating a flake --- src/nix/flake.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 470dfdc08..01385ff8d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -182,6 +182,51 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs } }; +struct CmdFlakeInit : virtual Args, Command +{ + std::string name() override + { + return "init"; + } + + std::string description() override + { + return "create a skeleton 'flake.nix' file in the current directory"; + } + + void run() override + { + Path flakeDir = absPath("."); + + if (!pathExists(flakeDir + "/.git")) + throw Error("the directory '%s' is not a Git repository", flakeDir); + + Path flakePath = flakeDir + "/flake.nix"; + + if (pathExists(flakePath)) + throw Error("file '%s' already exists", flakePath); + + writeFile(flakePath, +R"str( +{ + name = "hello"; + + description = "A flake for building Hello World"; + + epoch = 2019; + + requires = [ "nixpkgs" ]; + + provides = deps: rec { + + packages.hello = deps.nixpkgs.provides.packages.hello; + + }; +} +)str"); + } +}; + struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() @@ -190,7 +235,9 @@ struct CmdFlake : virtual MultiCommand, virtual Command , make_ref() , make_ref() , make_ref() - , make_ref()}) + , make_ref() + , make_ref() + }) { } From 507da65900ccb3c6356673e93ad2271c58e43b07 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 23:39:38 +0200 Subject: [PATCH 0102/2548] Move flake template into a separate file --- .gitignore | 2 +- src/nix/flake-template.nix | 15 +++++++++++++++ src/nix/flake.cc | 19 ++----------------- src/nix/local.mk | 2 ++ 4 files changed, 20 insertions(+), 18 deletions(-) create mode 100644 src/nix/flake-template.nix diff --git a/.gitignore b/.gitignore index b75c5d489..da0e7c843 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,7 @@ perl/Makefile.config /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/*.gen.hh +*.gen.* /src/nix/nix diff --git a/src/nix/flake-template.nix b/src/nix/flake-template.nix new file mode 100644 index 000000000..fe89e647e --- /dev/null +++ b/src/nix/flake-template.nix @@ -0,0 +1,15 @@ +{ + name = "hello"; + + description = "A flake for building Hello World"; + + epoch = 2019; + + requires = [ "nixpkgs" ]; + + provides = deps: rec { + + packages.hello = deps.nixpkgs.provides.packages.hello; + + }; +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 01385ff8d..3d2fb7832 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -207,23 +207,8 @@ struct CmdFlakeInit : virtual Args, Command throw Error("file '%s' already exists", flakePath); writeFile(flakePath, -R"str( -{ - name = "hello"; - - description = "A flake for building Hello World"; - - epoch = 2019; - - requires = [ "nixpkgs" ]; - - provides = deps: rec { - - packages.hello = deps.nixpkgs.provides.packages.hello; - - }; -} -)str"); +#include "flake-template.nix.gen.hh" + ); } }; diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d56..4003d0005 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -23,3 +23,5 @@ $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ $(eval $(call install-symlink, nix, $(bindir)/$(name)))) $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) + +$(d)/flake.cc: $(d)/flake-template.nix.gen.hh From c996e04aca2db1755ded4864465338afab677ff5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 23:47:29 +0200 Subject: [PATCH 0103/2548] Allow relative paths in flakerefs Also allow "." as an installable to refer to the flake in the current directory. E.g. $ nix build . will build 'provides.defaultPackage' in the flake in the current directory. --- flake.nix | 1 + src/libexpr/primops/flakeref.cc | 6 +++--- src/libexpr/primops/flakeref.hh | 2 +- src/nix/installables.cc | 14 ++++++++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index b119f0324..695f67fa4 100644 --- a/flake.nix +++ b/flake.nix @@ -14,5 +14,6 @@ packages.nix = hydraJobs.build.x86_64-linux; + defaultPackage = packages.nix; }; } diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 5f9a29260..1df53bfb8 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -32,7 +32,7 @@ const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*"; -FlakeRef::FlakeRef(const std::string & uri) +FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) { // FIXME: could combine this into one regex. @@ -106,9 +106,9 @@ FlakeRef::FlakeRef(const std::string & uri) data = d; } - else if (hasPrefix(uri, "/")) { + else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || uri == "."))) { IsPath d; - d.path = canonPath(uri); + d.path = allowRelative ? absPath(uri) : canonPath(uri); data = d; } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 832d7dd03..fa14f7c25 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -132,7 +132,7 @@ struct FlakeRef std::variant data; // Parse a flake URI. - FlakeRef(const std::string & uri); + FlakeRef(const std::string & uri, bool allowRelative = false); // Default constructor FlakeRef(const FlakeRef & flakeRef) : data(flakeRef.data) {}; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 631a849cd..f3be7b628 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -219,12 +219,18 @@ std::vector> SourceExprCommand::parseInstallables( else if ((colon = s.rfind(':')) != std::string::npos) { auto flakeRef = std::string(s, 0, colon); auto attrPath = std::string(s, colon + 1); - result.push_back(std::make_shared(*this, FlakeRef(flakeRef), attrPath)); + result.push_back(std::make_shared(*this, FlakeRef(flakeRef, true), attrPath)); } - else if (s.find('/') != std::string::npos) { - auto path = store->toStorePath(store->followLinksToStore(s)); - result.push_back(std::make_shared(path)); + else if (s.find('/') != std::string::npos || s == ".") { + Path storePath; + try { + storePath = store->toStorePath(store->followLinksToStore(s)); + } catch (Error) { } + if (storePath != "") + result.push_back(std::make_shared(storePath)); + else + result.push_back(std::make_shared(*this, FlakeRef(s, true), "defaultPackage")); } else From 87033f2c4e32f4851e8c2abf8ab3b56444b65590 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Apr 2019 23:58:33 +0200 Subject: [PATCH 0104/2548] Whitespace --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index f3be7b628..e792ce96d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -300,7 +300,7 @@ Path toStorePath(ref store, RealiseMode mode, auto paths = toStorePaths(store, mode, {installable}); if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + throw Error("argument '%s' should evaluate to one store path", installable->what()); return *paths.begin(); } From 18c019b616f457b1f9a39da8cafc012be5ddffcc Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0105/2548] Added nonFlakeRequires and the command `nix flake deps` --- src/libexpr/primops/flake.cc | 191 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 23 ++++- src/nix/build.cc | 14 +-- src/nix/flake.cc | 65 ++++++++++-- 4 files changed, 200 insertions(+), 93 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 7cfb2038c..c4ae29022 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -18,20 +18,19 @@ std::shared_ptr readRegistry(const Path & path) { auto registry = std::make_shared(); - try { - auto json = nlohmann::json::parse(readFile(path)); + if (!pathExists(path)) + return std::make_shared(); - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); + auto json = nlohmann::json::parse(readFile(path)); - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - registry->entries.emplace(i.key(), entry); - } - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; + registry->entries.emplace(i.key(), entry); } return registry; @@ -54,7 +53,6 @@ Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; } - std::shared_ptr getGlobalRegistry() { // FIXME: get from nixos.org. @@ -76,12 +74,20 @@ std::shared_ptr getFlagRegistry() const std::vector> EvalState::getFlakeRegistries() { std::vector> registries; - registries.push_back(getGlobalRegistry()); - registries.push_back(getUserRegistry()); + if (evalSettings.pureEval) { + registries.push_back(std::make_shared()); // global + registries.push_back(std::make_shared()); // user + registries.push_back(std::make_shared()); // local + } else { + registries.push_back(getGlobalRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getLocalRegistry()); + } registries.push_back(getFlagRegistry()); return registries; } +// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`. Value * makeFlakeRegistryValue(EvalState & state) { auto v = state.allocValue(); @@ -199,7 +205,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - auto sourceInfo = fetchFlake(state, flakeRef); + FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); @@ -209,18 +215,16 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (state.allowedPaths) state.allowedPaths->insert(flakePath); - FlakeRef newFlakeRef(flakeRef); - if (std::get_if(&newFlakeRef.data)) { - FlakeSourceInfo srcInfo = fetchFlake(state, newFlakeRef); - if (srcInfo.rev) { - std::string uri = flakeRef.baseRef().to_string(); - newFlakeRef = FlakeRef(uri + "/" + srcInfo.rev->to_string(Base16, false)); - } + Flake flake(flakeRef); + if (std::get_if(&flakeRef.data)) { + if (sourceInfo.rev) + flake.ref = FlakeRef(flakeRef.baseRef().to_string() + + "/" + sourceInfo.rev->to_string(Base16, false)); } - Flake flake(newFlakeRef); flake.path = flakePath; flake.revCount = sourceInfo.revCount; + flake.path = flakePath; Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -242,6 +246,15 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) *(**requires).value->listElems()[n], *(**requires).pos))); } + if (std::optional nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) { + state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos); + for (Attr attr : *(*(**nonFlakeRequires).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef); + } + } + if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { state.forceFunction(*(**provides).value, *(**provides).pos); flake.vProvides = (**provides).value; @@ -250,86 +263,107 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack - if (pathExists(lockFile)) { - flake.lockFile = readRegistry(lockFile); - for (auto & entry : flake.lockFile->entries) - if (!entry.second.ref.isImmutable()) - throw Error("flake lock file '%s' contains mutable entry '%s'", - lockFile, entry.second.ref.to_string()); - } + flake.lockFile = readRegistry(lockFile); + for (auto & entry : flake.lockFile->entries) + if (!entry.second.ref.isImmutable()) + throw Error("flake lock file '%s' contains mutable entry '%s'", + lockFile, entry.second.ref.to_string()); + return flake; } +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId) +{ + FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); + debug("got non-flake source '%s' with revision %s", + sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + + auto flakePath = sourceInfo.storePath; + state.store->assertStorePath(flakePath); + + if (state.allowedPaths) + state.allowedPaths->insert(flakePath); + + NonFlake nonFlake(flakeRef); + if (std::get_if(&flakeRef.data)) { + if (sourceInfo.rev) + nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string() + + "/" + sourceInfo.rev->to_string(Base16, false)); + } + + nonFlake.path = flakePath; + + nonFlake.id = flakeId; + + return nonFlake; +} + /* Given a flake reference, recursively fetch it and its dependencies. FIXME: this should return a graph of flakes. */ -static std::tuple> resolveFlake(EvalState & state, - const FlakeRef & topRef, bool impureTopRef) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) { - std::map done; - std::queue> todo; - std::optional topFlakeId; /// FIXME: ambiguous - todo.push({topRef, true}); + Dependencies deps; + std::queue todo; + bool isTopLevel = true; + todo.push(topRef); auto registries = state.getFlakeRegistries(); - //std::shared_ptr localRegistry = registries.at(2); while (!todo.empty()) { - auto [flakeRef, toplevel] = todo.front(); + auto flakeRef = todo.front(); todo.pop(); if (auto refData = std::get_if(&flakeRef.data)) { if (done.count(refData->id)) continue; // optimization - flakeRef = lookupFlake(state, flakeRef, - !evalSettings.pureEval || (toplevel && impureTopRef) ? registries : std::vector>()); - // This is why we need the `registries`. - } - -#if 0 - if (evalSettings.pureEval && !flakeRef.isImmutable() && (!toplevel || !impureTopRef)) + flakeRef = lookupFlake(state, flakeRef, registries); + if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); -#endif auto flake = getFlake(state, flakeRef); - if (done.count(flake.id)) continue; + if (isTopLevel) { + deps.topFlakeId = flake.id; + isTopLevel = false; + } - if (toplevel) topFlakeId = flake.id; + for (auto & flakeRef : flake.requires) + todo.push(flakeRef); - for (auto & require : flake.requires) - todo.push({require, false}); + for (auto & x : flake.nonFlakeRequires) + deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first)); + // TODO (Nick): If there are 2 non-flake dependencies with the same + // FlakeId, this will lead to trouble! One of the dependencies won't + // be used! -#if 0 - // The following piece of code basically adds the FlakeRefs from - // the lockfiles of dependencies to the localRegistry. This is used - // to resolve future `FlakeId`s, in `lookupFlake` a bit above this. - if (flake.lockFile) - for (auto & entry : flake.lockFile->entries) { - if (localRegistry->entries.count(entry.first)) continue; - localRegistry->entries.emplace(entry.first, entry.second); - } -#endif - - done.emplace(flake.id, std::move(flake)); + deps.flakes.push_back(flake); } - assert(topFlakeId); - return {*topFlakeId, std::move(done)}; + return deps; } FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) { FlakeRegistry newLockFile; - std::map myDependencyMap = get<1>(resolveFlake(evalState, flakeRef, false)); + Dependencies deps = resolveFlake(evalState, flakeRef, false); // Nick assumed that "topRefPure" means that the Flake for flakeRef can be // fetched purely. - for (auto const& require : myDependencyMap) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.second.ref); - // The FlakeRefs are immutable because they come out of the Flake objects, - // not from the requires. - newLockFile.entries.insert(std::pair(require.first, entry)); + for (auto const& require : deps.flakes) { + FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref); + // The FlakeRefs are immutable because they come out of the Flake objects. + if (require.id != deps.topFlakeId) + newLockFile.entries.insert_or_assign(require.id, entry); + // TODO (Nick): If there are 2 flake dependencies with the same FlakeId, + // one of them gets ignored! + } + for (auto const& nonFlake : deps.nonFlakes) { + FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref); + newLockFile.entries.insert_or_assign(nonFlake.id, entry); + // We are assuming the sets of FlakeIds for flakes and non-flakes + // are disjoint. } return newLockFile; } @@ -348,21 +382,26 @@ void updateLockFile(EvalState & state, std::string path) } } -Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) +// Return the `provides` of the top flake, while assigning to `v` the provides +// of the dependencies as well. +Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, Value & v) { - auto [topFlakeId, flakes] = resolveFlake(state, flakeRef, impureTopRef); + FlakeRef flakeRef = FlakeRef(flakeUri); + + Dependencies deps = resolveFlake(state, flakeRef, impure); // FIXME: we should call each flake with only its dependencies // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); + // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - state.mkAttrs(*vResult, flakes.size()); + state.mkAttrs(*vResult, dependencies.flakes.size()); Value * vTop = 0; - for (auto & flake : flakes) { - auto vFlake = state.allocAttr(*vResult, flake.second.id); + for (auto & flake : deps.flakes) { + auto vFlake = state.allocAttr(*vResult, flake.id); if (topFlakeId == flake.second.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index aea4e8aa2..ffd962561 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -38,15 +38,34 @@ struct Flake std::optional revCount; std::vector requires; std::shared_ptr lockFile; + std::map nonFlakeRequires; Value * vProvides; // FIXME: gc - // commit hash // date // content hash - Flake(FlakeRef & flakeRef) : ref(flakeRef) {}; + Flake(const FlakeRef flakeRef) : ref(flakeRef) {}; +}; + +struct NonFlake +{ + FlakeId id; + FlakeRef ref; + Path path; + // date + // content hash + NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; Flake getFlake(EvalState &, const FlakeRef &); +struct Dependencies +{ + FlakeId topFlakeId; + std::vector flakes; + std::vector nonFlakes; +}; + +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); + FlakeRegistry updateLockFile(EvalState &, Flake &); void updateLockFile(EvalState &, std::string); diff --git a/src/nix/build.cc b/src/nix/build.cc index da7c7f614..a6fcf5094 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,7 +11,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; - std::optional gitRepo = std::nullopt; + bool updateLock = true; CmdBuild() { @@ -28,9 +28,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand .set(&outLink, Path("")); mkFlag() - .longName("update-lock-file") - .description("update the lock file") - .dest(&gitRepo); + .longName("no-update") + .description("don't update the lock file") + .set(&updateLock, false); } std::string name() override @@ -78,8 +78,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - if (gitRepo) - updateLockFile(*evalState, *gitRepo); + if(updateLock) + for (int i = 0; i < installables.size(); i++) + if (auto flakeUri = installableToFlakeUri) + updateLockFile(*evalState, FlakeRef(*flakeUri)); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3d2fb7832..07d31c45a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -36,6 +36,60 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs } }; +void printFlakeInfo(Flake & flake, bool json) { + if (json) { + nlohmann::json j; + j["name"] = flake.id; + j["location"] = flake.path; + j["description"] = flake.description; + std::cout << j.dump(4) << std::endl; + } else { + std::cout << "Name: " << flake.id << "\n"; + std::cout << "Description: " << flake.description << "\n"; + std::cout << "Location: " << flake.path << "\n"; + } +} + +void printNonFlakeInfo(NonFlake & nonFlake, bool json) { + if (json) { + nlohmann::json j; + j["name"] = nonFlake.id; + j["location"] = nonFlake.path; + std::cout << j.dump(4) << std::endl; + } else { + std::cout << "name: " << nonFlake.id << "\n"; + std::cout << "Location: " << nonFlake.path << "\n"; + } +} + +struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs +{ + std::string name() override + { + return "deps"; + } + + std::string description() override + { + return "list informaton about dependencies"; + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + FlakeRef flakeRef(flakeUri); + + Dependencies deps = resolveFlake(*evalState, flakeRef, true); + + for (auto & flake : deps.flakes) + printFlakeInfo(flake, json); + + for (auto & nonFlake : deps.nonFlakes) + printNonFlakeInfo(nonFlake, json); + } +}; + struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs { std::string name() override @@ -73,15 +127,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand { auto evalState = std::make_shared(searchPath, store); nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); - if (json) { - nlohmann::json j; - j["location"] = flake.path; - j["description"] = flake.description; - std::cout << j.dump(4) << std::endl; - } else { - std::cout << "Description: " << flake.description << "\n"; - std::cout << "Location: " << flake.path << "\n"; - } + printFlakeInfo(flake, json); } }; @@ -218,6 +264,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command : MultiCommand({make_ref() , make_ref() , make_ref() + , make_ref() , make_ref() , make_ref() , make_ref() From 641db127be9df82fe4d51290120a8ba6d0b5f4fd Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0106/2548] FlakeIds are now properly looked up in registries --- src/libexpr/primops/flake.cc | 23 +++++++++++++---------- src/libexpr/primops/flake.hh | 2 +- src/libexpr/primops/flakeref.hh | 18 ++++++++++++++++++ src/nix/build.cc | 7 ++++--- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c4ae29022..70d1b871a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -123,6 +123,10 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); + if (refData->ref) + newRef.setRef(*refData->ref); + if (refData->rev) + newRef.setRev(*refData->rev); return newRef; } } @@ -224,7 +228,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) flake.path = flakePath; flake.revCount = sourceInfo.revCount; - flake.path = flakePath; Value vInfo; state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack @@ -317,9 +320,9 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impur auto flakeRef = todo.front(); todo.pop(); - if (auto refData = std::get_if(&flakeRef.data)) { - if (done.count(refData->id)) continue; // optimization + if (std::get_if(&flakeRef.data)) flakeRef = lookupFlake(state, flakeRef, registries); + if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); @@ -368,7 +371,7 @@ FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) return newLockFile; } -void updateLockFile(EvalState & state, std::string path) +void updateLockFile(EvalState & state, Path path) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); @@ -384,7 +387,7 @@ void updateLockFile(EvalState & state, std::string path) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, Value & v) +Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) { FlakeRef flakeRef = FlakeRef(flakeUri); @@ -406,16 +409,16 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, bool impureTopRef, state.mkAttrs(*vFlake, 4); - mkString(*state.allocAttr(*vFlake, state.sDescription), flake.second.description); + mkString(*state.allocAttr(*vFlake, state.sDescription), flake.description); - state.store->assertStorePath(flake.second.path); - mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.second.path, {flake.second.path}); + state.store->assertStorePath(flake.path); + mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path}); if (flake.second.revCount) - mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.second.revCount); + mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); - mkApp(*vProvides, *flake.second.vProvides, *vResult); + mkApp(*vProvides, *flake.vProvides, *vResult); // Should this be vResult or vFlake??? Or both! vFlake->attrs->sort(); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index ffd962561..a8f907784 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -68,5 +68,5 @@ Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); FlakeRegistry updateLockFile(EvalState &, Flake &); -void updateLockFile(EvalState &, std::string); +void updateLockFile(EvalState &, Path); } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index fa14f7c25..94a75fb2b 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -160,5 +160,23 @@ struct FlakeRef bool isImmutable() const; FlakeRef baseRef() const; + + void setRef(std::optional ref) { + if (auto refData = std::get_if(&data)) + refData->ref = ref; + else if (auto refData = std::get_if(&data)) + refData->ref = ref; + else if (auto refData = std::get_if(&data)) + refData->ref = ref; + } + + void setRev(std::optional rev) { + if (auto refData = std::get_if(&data)) + refData->rev = rev; + else if (auto refData = std::get_if(&data)) + refData->rev = rev; + else if (auto refData = std::get_if(&data)) + refData->rev = rev; + } }; } diff --git a/src/nix/build.cc b/src/nix/build.cc index a6fcf5094..f6908b0c0 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,10 +78,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } + std::string flakeUri = ""; if(updateLock) - for (int i = 0; i < installables.size(); i++) - if (auto flakeUri = installableToFlakeUri) - updateLockFile(*evalState, FlakeRef(*flakeUri)); + for (uint i = 0; i < installables.size(); i++) + // if (auto flakeUri = installableToFlakeUri) + updateLockFile(*evalState, flakeUri); } }; From 3ec0c82fab94533807c5c3bb25df2b43d8339ed3 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Fri, 29 Mar 2019 16:18:25 +0100 Subject: [PATCH 0107/2548] Fixed dependency resolution --- src/libexpr/primops/flake.cc | 204 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 31 ++++-- src/nix/build.cc | 10 +- src/nix/flake.cc | 25 +++-- 4 files changed, 168 insertions(+), 102 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 70d1b871a..1ad9ad3f8 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -39,9 +39,8 @@ std::shared_ptr readRegistry(const Path & path) /* Write the registry or lock file to a file. */ void writeRegistry(FlakeRegistry registry, Path path) { - nlohmann::json json = {}; + nlohmann::json json; json["version"] = 1; - json["flakes"] = {}; for (auto elem : registry.entries) { json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; } @@ -49,6 +48,85 @@ void writeRegistry(FlakeRegistry registry, Path path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } +LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + + LockFile::FlakeEntry entry(flakeRef); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + for (auto & x : entry.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : entry.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + return json; +} + +void writeLockFile(LockFile lockFile, Path path) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeRequires"]; + for (auto & x : lockFile.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // '4' = indentation in json file +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -142,17 +220,13 @@ struct FlakeSourceInfo std::optional revCount; }; -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeRef directFlakeRef = FlakeRef(flakeRef); - if (!flakeRef.isDirect()) { - directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); - } - assert(directFlakeRef.isDirect()); - // NOTE FROM NICK: I don't see why one wouldn't fetch FlakeId flakes.. + FlakeRef directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); if (auto refData = std::get_if(&directFlakeRef.data)) { - // FIXME: require hash in pure mode. + if (evalSettings.pureEval && !impureIsAllowed && !directFlakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", directFlakeRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -207,7 +281,8 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef) else abort(); } -Flake getFlake(EvalState & state, const FlakeRef & flakeRef) +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with revision %s", @@ -264,14 +339,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake lacks attribute 'provides'"); - auto lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack - - flake.lockFile = readRegistry(lockFile); - for (auto & entry : flake.lockFile->entries) - if (!entry.second.ref.isImmutable()) - throw Error("flake lock file '%s' contains mutable entry '%s'", - lockFile, entry.second.ref.to_string()); + const Path lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack + flake.lockFile = readLockFile(lockFile); return flake; } @@ -307,77 +377,50 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flake dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake = true) { - Dependencies deps; - std::queue todo; - bool isTopLevel = true; - todo.push(topRef); + Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); + Dependencies deps(flake); - auto registries = state.getFlakeRegistries(); + for (auto & nonFlakeInfo : flake.nonFlakeRequires) + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); - while (!todo.empty()) { - auto flakeRef = todo.front(); - todo.pop(); - - if (std::get_if(&flakeRef.data)) - flakeRef = lookupFlake(state, flakeRef, registries); - - if (evalSettings.pureEval && !flakeRef.isImmutable() && (!isTopLevel || !impureTopRef)) - throw Error("mutable flake '%s' is not allowed in pure mode; use --impure to disable", flakeRef.to_string()); - - auto flake = getFlake(state, flakeRef); - - if (isTopLevel) { - deps.topFlakeId = flake.id; - isTopLevel = false; - } - - for (auto & flakeRef : flake.requires) - todo.push(flakeRef); - - for (auto & x : flake.nonFlakeRequires) - deps.nonFlakes.push_back(getNonFlake(state, x.second, x.first)); - // TODO (Nick): If there are 2 non-flake dependencies with the same - // FlakeId, this will lead to trouble! One of the dependencies won't - // be used! - - deps.flakes.push_back(flake); - } + for (auto & newFlakeRef : flake.requires) + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, impureTopRef, false)); return deps; } -FlakeRegistry updateLockFile(EvalState & evalState, FlakeRef & flakeRef) +LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) { - FlakeRegistry newLockFile; - Dependencies deps = resolveFlake(evalState, flakeRef, false); - // Nick assumed that "topRefPure" means that the Flake for flakeRef can be - // fetched purely. - for (auto const& require : deps.flakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(require.ref); - // The FlakeRefs are immutable because they come out of the Flake objects. - if (require.id != deps.topFlakeId) - newLockFile.entries.insert_or_assign(require.id, entry); - // TODO (Nick): If there are 2 flake dependencies with the same FlakeId, - // one of them gets ignored! - } - for (auto const& nonFlake : deps.nonFlakes) { - FlakeRegistry::Entry entry = FlakeRegistry::Entry(nonFlake.ref); - newLockFile.entries.insert_or_assign(nonFlake.id, entry); - // We are assuming the sets of FlakeIds for flakes and non-flakes - // are disjoint. - } - return newLockFile; + LockFile::FlakeEntry entry(deps.flake.ref); + + for (Dependencies & deps : deps.flakeDeps) + entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); + + for (NonFlake & nonFlake : deps.nonFlakeDeps) + entry.nonFlakeEntries.insert_or_assign(nonFlake.id, nonFlake.ref); + + return entry; } -void updateLockFile(EvalState & state, Path path) +LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopRef) +{ + Dependencies deps = resolveFlake(evalState, flakeRef, impureTopRef); + LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); + LockFile lockFile; + lockFile.flakeEntries = entry.flakeEntries; + lockFile.nonFlakeEntries = entry.nonFlakeEntries; + return lockFile; +} + +void updateLockFile(EvalState & state, Path path, bool impureTopRef) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { - FlakeRegistry newLockFile = updateLockFile(state, flakeRef); - writeRegistry(newLockFile, path + "/flake.lock"); + LockFile lockFile = getLockFile(state, flakeRef, impureTopRef); + writeLockFile(lockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { @@ -393,19 +436,19 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) Dependencies deps = resolveFlake(state, flakeRef, impure); - // FIXME: we should call each flake with only its dependencies - // (rather than the closure of the top-level flake). + // // FIXME: we should call each flake with only its dependencies + // // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - state.mkAttrs(*vResult, dependencies.flakes.size()); + state.mkAttrs(*vResult, deps.flakeDeps.size()); Value * vTop = 0; - for (auto & flake : deps.flakes) { + for (auto & flake : deps.flakeDeps) { auto vFlake = state.allocAttr(*vResult, flake.id); - if (topFlakeId == flake.second.id) vTop = vFlake; + if (deps.topFlakeId == flake.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); @@ -431,6 +474,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) return vTop; } +// This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index a8f907784..019688f37 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -19,6 +19,20 @@ struct FlakeRegistry std::map entries; }; +struct LockFile +{ + struct FlakeEntry + { + FlakeRef ref; + std::map flakeEntries; + std::map nonFlakeEntries; + FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; + }; + + std::map flakeEntries; + std::map nonFlakeEntries; +}; + Path getUserRegistryPath(); Value * makeFlakeRegistryValue(EvalState & state); @@ -37,7 +51,7 @@ struct Flake Path path; std::optional revCount; std::vector requires; - std::shared_ptr lockFile; + LockFile lockFile; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc // date @@ -55,18 +69,17 @@ struct NonFlake NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; -Flake getFlake(EvalState &, const FlakeRef &); +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct Dependencies { - FlakeId topFlakeId; - std::vector flakes; - std::vector nonFlakes; + Flake flake; + std::vector flakeDeps; // The flake dependencies + std::vector nonFlakeDeps; + Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef); +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); -FlakeRegistry updateLockFile(EvalState &, Flake &); - -void updateLockFile(EvalState &, Path); +void updateLockFile(EvalState &, Path path, bool impureTopRef); } diff --git a/src/nix/build.cc b/src/nix/build.cc index f6908b0c0..226c21e9e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -78,11 +78,11 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - std::string flakeUri = ""; - if(updateLock) - for (uint i = 0; i < installables.size(); i++) - // if (auto flakeUri = installableToFlakeUri) - updateLockFile(*evalState, flakeUri); + // std::string flakeUri = ""; + // if(updateLock) + // for (uint i = 0; i < installables.size(); i++) + // // if (auto flakeUri = installableToFlakeUri) + // updateLockFile(*evalState, flakeUri, true); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 07d31c45a..ff291aa80 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -5,6 +5,7 @@ #include "progress-bar.hh" #include "eval.hh" #include +#include using namespace nix; @@ -80,13 +81,21 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, true, true); - for (auto & flake : deps.flakes) - printFlakeInfo(flake, json); + std::queue todo; + todo.push(deps); - for (auto & nonFlake : deps.nonFlakes) - printNonFlakeInfo(nonFlake, json); + while (!todo.empty()) { + deps = todo.front(); + todo.pop(); + + for (auto & nonFlake : deps.nonFlakeDeps) + printNonFlakeInfo(nonFlake, json); + + for (auto & newDeps : deps.flakeDeps) + todo.push(newDeps); + } } }; @@ -107,7 +116,7 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs auto evalState = std::make_shared(searchPath, store); if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath); + updateLockFile(*evalState, gitPath, true); } }; @@ -126,7 +135,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; @@ -220,7 +229,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs auto it = userRegistry.entries.find(flakeId); if (it != userRegistry.entries.end()) { FlakeRef oldRef = it->second.ref; - it->second.ref = getFlake(*evalState, oldRef).ref; + it->second.ref = getFlake(*evalState, oldRef, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else From f39670c6318ba8d2260b3ac54f46161d74649266 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Sat, 6 Apr 2019 20:45:35 +0200 Subject: [PATCH 0108/2548] Took ref and rev out of FlakeRef --- src/libexpr/primops/flake.cc | 31 +++++++-------- src/libexpr/primops/flakeref.cc | 69 +++++++++++---------------------- src/libexpr/primops/flakeref.hh | 28 +++---------- 3 files changed, 41 insertions(+), 87 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1ad9ad3f8..cea3854e4 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -201,10 +201,8 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); - if (refData->ref) - newRef.setRef(*refData->ref); - if (refData->rev) - newRef.setRev(*refData->rev); + if (flakeRef.ref) newRef.setRef(*flakeRef.ref); + if (flakeRef.rev) newRef.setRev(*flakeRef.rev); return newRef; } } @@ -220,13 +218,13 @@ struct FlakeSourceInfo std::optional revCount; }; -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) { - FlakeRef directFlakeRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); + FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); - if (auto refData = std::get_if(&directFlakeRef.data)) { - if (evalSettings.pureEval && !impureIsAllowed && !directFlakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", directFlakeRef.to_string()); + if (auto refData = std::get_if(&fRef.data)) { + if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. @@ -235,14 +233,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo, - refData->rev - ? refData->rev->to_string(Base16, false) - : refData->ref - ? *refData->ref - : "master"); + fRef.rev ? fRef.rev->to_string(Base16, false) + : fRef.ref ? *fRef.ref : "master"); auto result = getDownloader()->downloadCached(state.store, url, true, "source", - Hash(), nullptr, refData->rev ? 1000000000 : settings.tarballTtl); + Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); @@ -257,9 +252,9 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, return info; } - else if (auto refData = std::get_if(&directFlakeRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, refData->ref, - refData->rev ? refData->rev->to_string(Base16, false) : "", "source"); + else if (auto refData = std::get_if(&fRef.data)) { + auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, + fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); FlakeSourceInfo info; info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 1df53bfb8..f160b257b 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -58,11 +58,11 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) IsFlakeId d; d.id = match[1]; if (match[2].matched) - d.rev = Hash(match[2], htSHA1); + rev = Hash(match[2], htSHA1); else if (match[3].matched) { - d.ref = match[3]; + ref = match[3]; if (match[4].matched) - d.rev = Hash(match[4], htSHA1); + rev = Hash(match[4], htSHA1); } data = d; } @@ -72,9 +72,9 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) d.owner = match[1]; d.repo = match[2]; if (match[3].matched) - d.rev = Hash(match[3], htSHA1); + rev = Hash(match[3], htSHA1); else if (match[4].matched) { - d.ref = match[4]; + ref = match[4]; } data = d; } @@ -92,16 +92,16 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) if (name == "rev") { if (!std::regex_match(value, revRegex)) throw Error("invalid Git revision '%s'", value); - d.rev = Hash(value, htSHA1); + rev = Hash(value, htSHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex2)) throw Error("invalid Git ref '%s'", value); - d.ref = value; + ref = value; } else // FIXME: should probably pass through unknown parameters throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); } - if (d.rev && !d.ref) + if (rev && !ref) throw Error("flake URI '%s' lacks a Git ref", uri); data = d; } @@ -118,27 +118,18 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { - if (auto refData = std::get_if(&data)) { - return - "flake:" + refData->id + - (refData->ref ? "/" + *refData->ref : "") + - (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); - } + std::string string; + if (auto refData = std::get_if(&data)) + string = "flake:" + refData->id; else if (auto refData = std::get_if(&data)) { - assert(!refData->ref || !refData->rev); - return - "github:" + refData->owner + "/" + refData->repo + - (refData->ref ? "/" + *refData->ref : "") + - (refData->rev ? "/" + refData->rev->to_string(Base16, false) : ""); + assert(!ref || !rev); + string = "github:" + refData->owner + "/" + refData->repo; } else if (auto refData = std::get_if(&data)) { - assert(refData->ref || !refData->rev); - return - refData->uri + - (refData->ref ? "?ref=" + *refData->ref : "") + - (refData->rev ? "&rev=" + refData->rev->to_string(Base16, false) : ""); + assert(ref || !rev); + string = refData->uri; } else if (auto refData = std::get_if(&data)) { @@ -146,38 +137,22 @@ std::string FlakeRef::to_string() const } else abort(); + + string += (ref ? "/" + *ref : "") + + (rev ? "/" + rev->to_string(Base16, false) : ""); + return string; } bool FlakeRef::isImmutable() const { - if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (auto refData = std::get_if(&data)) - return (bool) refData->rev; - - else if (std::get_if(&data)) - return false; - - else abort(); + return (bool) rev; } FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. { FlakeRef result(*this); - if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } else if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } else if (auto refData = std::get_if(&result.data)) { - refData->ref = std::nullopt; - refData->rev = std::nullopt; - } + result.ref = std::nullopt; + result.rev = std::nullopt; return result; } } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 94a75fb2b..9276fc737 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -101,25 +101,23 @@ typedef std::string FlakeId; struct FlakeRef { + std::optional ref; + std::optional rev; + struct IsFlakeId { FlakeId id; - std::optional ref; - std::optional rev; }; struct IsGitHub { std::string owner, repo; - std::optional ref; - std::optional rev; }; + // Git, Tarball struct IsGit { std::string uri; - std::optional ref; - std::optional rev; }; struct IsPath @@ -161,22 +159,8 @@ struct FlakeRef FlakeRef baseRef() const; - void setRef(std::optional ref) { - if (auto refData = std::get_if(&data)) - refData->ref = ref; - else if (auto refData = std::get_if(&data)) - refData->ref = ref; - else if (auto refData = std::get_if(&data)) - refData->ref = ref; - } + void setRef(std::optional ref) { ref = ref; } - void setRev(std::optional rev) { - if (auto refData = std::get_if(&data)) - refData->rev = rev; - else if (auto refData = std::get_if(&data)) - refData->rev = rev; - else if (auto refData = std::get_if(&data)) - refData->rev = rev; - } + void setRev(std::optional rev) { rev = rev; } }; } From c64f98b883515df70e2457ae01070b5af9ae69b9 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0109/2548] FlakeAlias is implemented --- src/libexpr/primops/flake.cc | 104 ++++++++++++++++++++++++++++---- src/libexpr/primops/flake.hh | 10 +-- src/libexpr/primops/flakeref.cc | 15 +++-- src/libexpr/primops/flakeref.hh | 8 ++- src/nix/build.cc | 2 +- src/nix/flake.cc | 36 +++++------ 6 files changed, 129 insertions(+), 46 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index cea3854e4..729b1da95 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -127,6 +127,86 @@ void writeLockFile(LockFile lockFile, Path path) writeFile(path, json.dump(4)); // '4' = indentation in json file } +Path getUserRegistryPath() +>>>>>>> Fixed dependency resolution +{ + FlakeRef flakeRef(json["uri"]); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + + LockFile::FlakeEntry entry(flakeRef); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return entry; +} + +LockFile readLockFile(const Path & path) +{ + LockFile lockFile; + + if (!pathExists(path)) + return lockFile; + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("lock file '%s' has unsupported version %d", path, version); + + auto nonFlakeRequires = json["nonFlakeRequires"]; + + for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + FlakeRef flakeRef(i->value("uri", "")); + if (!flakeRef.isImmutable()) + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + } + + auto requires = json["requires"]; + + for (auto i = requires.begin(); i != requires.end(); ++i) + lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + + return lockFile; +} + +nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +{ + nlohmann::json json; + json["uri"] = entry.ref.to_string(); + for (auto & x : entry.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : entry.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + return json; +} + +void writeLockFile(LockFile lockFile, Path path) +{ + nlohmann::json json; + json["version"] = 1; + json["nonFlakeRequires"]; + for (auto & x : lockFile.nonFlakeEntries) + json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.flakeEntries) + json["requires"][x.first] = flakeEntryToJson(x.second); + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // '4' = indentation in json file +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -194,9 +274,9 @@ Value * makeFlakeRegistryValue(EvalState & state) static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, std::vector> registries) { - if (auto refData = std::get_if(&flakeRef.data)) { + if (auto refData = std::get_if(&flakeRef.data)) { for (auto registry : registries) { - auto i = registry->entries.find(refData->id); + auto i = registry->entries.find(refData->alias); if (i != registry->entries.end()) { auto newRef = FlakeRef(i->second.ref); if (!newRef.isDirect()) @@ -206,7 +286,7 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return newRef; } } - throw Error("cannot find flake '%s' in the flake registry or in the flake lock file", refData->id); + throw Error("cannot find flake with alias '%s' in the flake registry or in the flake lock file", refData->alias); } else return flakeRef; } @@ -342,7 +422,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } // Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flakeId) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias) { FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got non-flake source '%s' with revision %s", @@ -363,7 +443,7 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeId flake nonFlake.path = flakePath; - nonFlake.id = flakeId; + nonFlake.alias = alias; return nonFlake; } @@ -394,14 +474,14 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); for (NonFlake & nonFlake : deps.nonFlakeDeps) - entry.nonFlakeEntries.insert_or_assign(nonFlake.id, nonFlake.ref); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); return entry; } -LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopRef) +LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef) { - Dependencies deps = resolveFlake(evalState, flakeRef, impureTopRef); + Dependencies deps = resolveFlake(evalState, flakeRef, true); LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); LockFile lockFile; lockFile.flakeEntries = entry.flakeEntries; @@ -409,17 +489,17 @@ LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef, bool impureTopR return lockFile; } -void updateLockFile(EvalState & state, Path path, bool impureTopRef) +void updateLockFile(EvalState & state, Path path) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); if (std::get_if(&flakeRef.data)) { - LockFile lockFile = getLockFile(state, flakeRef, impureTopRef); + LockFile lockFile = getLockFile(state, flakeRef); writeLockFile(lockFile, path + "/flake.lock"); } else if (std::get_if(&flakeRef.data)) { throw UsageError("you can only update local flakes, not flakes on GitHub"); } else { - throw UsageError("you can only update local flakes, not flakes through their FlakeId"); + throw UsageError("you can only update local flakes, not flakes through their FlakeAlias"); } } @@ -456,7 +536,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); - mkApp(*vProvides, *flake.vProvides, *vResult); // Should this be vResult or vFlake??? Or both! + mkApp(*vProvides, *flake.vProvides, *vResult); vFlake->attrs->sort(); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 019688f37..adf8b07af 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -16,7 +16,7 @@ struct FlakeRegistry Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; Entry operator=(const Entry & entry) { return Entry(entry.ref); } }; - std::map entries; + std::map entries; }; struct LockFile @@ -52,7 +52,7 @@ struct Flake std::optional revCount; std::vector requires; LockFile lockFile; - std::map nonFlakeRequires; + std::map nonFlakeRequires; Value * vProvides; // FIXME: gc // date // content hash @@ -61,7 +61,7 @@ struct Flake struct NonFlake { - FlakeId id; + FlakeAlias alias; FlakeRef ref; Path path; // date @@ -81,5 +81,7 @@ struct Dependencies Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); -void updateLockFile(EvalState &, Path path, bool impureTopRef); +FlakeRegistry updateLockFile(EvalState &, Flake &); + +void updateLockFile(EvalState &, Path path); } diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index f160b257b..ab1e5e152 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -19,7 +19,7 @@ const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + // "master/e72daba8250068216d79d2aeef40d4d95aff6666"). const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; // GitHub references. const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; @@ -37,7 +37,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) // FIXME: could combine this into one regex. static std::regex flakeRegex( - "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?", + "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", std::regex::ECMAScript); static std::regex githubRegex( @@ -55,8 +55,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::cmatch match; if (std::regex_match(uri.c_str(), match, flakeRegex)) { - IsFlakeId d; - d.id = match[1]; + IsAlias d; + d.alias = match[1]; if (match[2].matched) rev = Hash(match[2], htSHA1); else if (match[3].matched) { @@ -119,8 +119,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { std::string string; - if (auto refData = std::get_if(&data)) - string = "flake:" + refData->id; + if (auto refData = std::get_if(&data)) + string = "flake:" + refData->alias; else if (auto refData = std::get_if(&data)) { assert(!ref || !rev); @@ -132,9 +132,8 @@ std::string FlakeRef::to_string() const string = refData->uri; } - else if (auto refData = std::get_if(&data)) { + else if (auto refData = std::get_if(&data)) return refData->path; - } else abort(); diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 9276fc737..32904953a 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -98,15 +98,17 @@ namespace nix { */ typedef std::string FlakeId; +typedef std::string FlakeAlias; +typedef std::string FlakeUri; struct FlakeRef { std::optional ref; std::optional rev; - struct IsFlakeId + struct IsAlias { - FlakeId id; + FlakeAlias alias; }; struct IsGitHub @@ -150,7 +152,7 @@ struct FlakeRef a flake ID, which requires a lookup in the flake registry. */ bool isDirect() const { - return !std::get_if(&data); + return !std::get_if(&data); } /* Check whether this is an "immutable" flake reference, that is, diff --git a/src/nix/build.cc b/src/nix/build.cc index 226c21e9e..a2fc56e69 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -82,7 +82,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand // if(updateLock) // for (uint i = 0; i < installables.size(); i++) // // if (auto flakeUri = installableToFlakeUri) - // updateLockFile(*evalState, flakeUri, true); + // updateLockFile(*evalState, flakeUri); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ff291aa80..df944a148 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -40,12 +40,12 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs void printFlakeInfo(Flake & flake, bool json) { if (json) { nlohmann::json j; - j["name"] = flake.id; + j["id"] = flake.id; j["location"] = flake.path; j["description"] = flake.description; std::cout << j.dump(4) << std::endl; } else { - std::cout << "Name: " << flake.id << "\n"; + std::cout << "ID: " << flake.id << "\n"; std::cout << "Description: " << flake.description << "\n"; std::cout << "Location: " << flake.path << "\n"; } @@ -54,11 +54,11 @@ void printFlakeInfo(Flake & flake, bool json) { void printNonFlakeInfo(NonFlake & nonFlake, bool json) { if (json) { nlohmann::json j; - j["name"] = nonFlake.id; + j["name"] = nonFlake.alias; j["location"] = nonFlake.path; std::cout << j.dump(4) << std::endl; } else { - std::cout << "name: " << nonFlake.id << "\n"; + std::cout << "name: " << nonFlake.alias << "\n"; std::cout << "Location: " << nonFlake.path << "\n"; } } @@ -116,7 +116,7 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs auto evalState = std::make_shared(searchPath, store); if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath, true); + updateLockFile(*evalState, gitPath); } }; @@ -135,15 +135,15 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); printFlakeInfo(flake, json); } }; struct CmdFlakeAdd : MixEvalArgs, Command { - std::string flakeId; - std::string flakeUri; + FlakeAlias flakeAlias; + FlakeUri flakeUri; std::string name() override { @@ -157,7 +157,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command CmdFlakeAdd() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); expectArg("flake-uri", &flakeUri); } @@ -167,15 +167,15 @@ struct CmdFlakeAdd : MixEvalArgs, Command Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); FlakeRegistry::Entry entry(newFlakeRef); - userRegistry->entries.erase(flakeId); - userRegistry->entries.insert_or_assign(flakeId, newFlakeRef); + userRegistry->entries.erase(flakeAlias); + userRegistry->entries.insert_or_assign(flakeAlias, newFlakeRef); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command { - std::string flakeId; + FlakeAlias flakeAlias; std::string name() override { @@ -189,21 +189,21 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); } void run() override { Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - userRegistry->entries.erase(flakeId); + userRegistry->entries.erase(flakeAlias); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs { - std::string flakeId; + FlakeAlias flakeAlias; std::string name() override { @@ -217,7 +217,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { - expectArg("flake-id", &flakeId); + expectArg("flake-id", &flakeAlias); } void run(nix::ref store) override @@ -226,14 +226,14 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs Path userRegistryPath = getUserRegistryPath(); FlakeRegistry userRegistry = *readRegistry(userRegistryPath); - auto it = userRegistry.entries.find(flakeId); + auto it = userRegistry.entries.find(flakeAlias); if (it != userRegistry.entries.end()) { FlakeRef oldRef = it->second.ref; it->second.ref = getFlake(*evalState, oldRef, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else - throw Error("the flake identifier '%s' does not exist in the user registry", flakeId); + throw Error("the flake alias '%s' does not exist in the user registry", flakeAlias); } }; From 4ad4e4866891a62a6e1bb919d81e224ba0a1cf1c Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Mon, 8 Apr 2019 19:03:00 +0200 Subject: [PATCH 0110/2548] FlakeRegistry = FlakeRef -> FlakeRef --- flake-registry.json | 4 +- src/libexpr/primops/flake.cc | 169 +++++++++----------------------- src/libexpr/primops/flake.hh | 10 +- src/libexpr/primops/flakeref.hh | 50 ++++++---- src/nix/build.cc | 3 +- src/nix/command.hh | 3 +- src/nix/flake.cc | 45 ++++----- src/nix/installables.cc | 1 - 8 files changed, 105 insertions(+), 180 deletions(-) diff --git a/flake-registry.json b/flake-registry.json index b850daa74..378290ec6 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -1,5 +1,4 @@ { - "version": 1, "flakes": { "dwarffs": { "uri": "github:edolstra/dwarffs/flake" @@ -7,5 +6,6 @@ "nixpkgs": { "uri": "github:edolstra/nixpkgs/flake" } - } + }, + "version": 1 } diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 729b1da95..145d79446 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -28,10 +28,8 @@ std::shared_ptr readRegistry(const Path & path) throw Error("flake registry '%s' has unsupported version %d", path, version); auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - FlakeRegistry::Entry entry{FlakeRef(i->value("uri", ""))}; - registry->entries.emplace(i.key(), entry); - } + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); return registry; } @@ -41,9 +39,8 @@ void writeRegistry(FlakeRegistry registry, Path path) { nlohmann::json json; json["version"] = 1; - for (auto elem : registry.entries) { - json["flakes"][elem.first] = { {"uri", elem.second.ref.to_string()} }; - } + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } @@ -127,106 +124,31 @@ void writeLockFile(LockFile lockFile, Path path) writeFile(path, json.dump(4)); // '4' = indentation in json file } -Path getUserRegistryPath() ->>>>>>> Fixed dependency resolution +std::shared_ptr getGlobalRegistry() { - FlakeRef flakeRef(json["uri"]); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - - LockFile::FlakeEntry entry(flakeRef); - - auto nonFlakeRequires = json["nonFlakeRequires"]; - - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); - } - - auto requires = json["requires"]; - - for (auto i = requires.begin(); i != requires.end(); ++i) - entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return entry; -} - -LockFile readLockFile(const Path & path) -{ - LockFile lockFile; - - if (!pathExists(path)) - return lockFile; - - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("lock file '%s' has unsupported version %d", path, version); - - auto nonFlakeRequires = json["nonFlakeRequires"]; - - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); - } - - auto requires = json["requires"]; - - for (auto i = requires.begin(); i != requires.end(); ++i) - lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return lockFile; -} - -nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) -{ - nlohmann::json json; - json["uri"] = entry.ref.to_string(); - for (auto & x : entry.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); - for (auto & x : entry.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); - return json; -} - -void writeLockFile(LockFile lockFile, Path path) -{ - nlohmann::json json; - json["version"] = 1; - json["nonFlakeRequires"]; - for (auto & x : lockFile.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); - for (auto & x : lockFile.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); - createDirs(dirOf(path)); - writeFile(path, json.dump(4)); // '4' = indentation in json file + return std::make_shared(); } Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; } -std::shared_ptr getGlobalRegistry() -{ - // FIXME: get from nixos.org. - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); -} std::shared_ptr getUserRegistry() { return readRegistry(getUserRegistryPath()); } +std::shared_ptr getLocalRegistry() +{ + Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; + return readRegistry(registryFile); +} + std::shared_ptr getFlagRegistry() { + // TODO (Nick): Implement this. return std::make_shared(); - // TODO: Implement this once the right flags are implemented. } const std::vector> EvalState::getFlakeRegistries() @@ -259,9 +181,9 @@ Value * makeFlakeRegistryValue(EvalState & state) for (auto & registry : registries) { for (auto & entry : registry->entries) { - auto vEntry = state.allocAttr(*v, entry.first); + auto vEntry = state.allocAttr(*v, entry.first.to_string()); state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.ref.to_string()); + mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.to_string()); vEntry->attrs->sort(); } } @@ -272,23 +194,30 @@ Value * makeFlakeRegistryValue(EvalState & state) } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, - std::vector> registries) + std::vector> registries, std::vector pastSearches = {}) { - if (auto refData = std::get_if(&flakeRef.data)) { - for (auto registry : registries) { - auto i = registry->entries.find(refData->alias); - if (i != registry->entries.end()) { - auto newRef = FlakeRef(i->second.ref); - if (!newRef.isDirect()) - throw Error("found indirect flake URI '%s' in the flake registry", i->second.ref.to_string()); - if (flakeRef.ref) newRef.setRef(*flakeRef.ref); - if (flakeRef.rev) newRef.setRev(*flakeRef.rev); - return newRef; + for (std::shared_ptr registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + if (std::get_if(&flakeRef.data)) { + if (flakeRef.ref) newRef.ref = flakeRef.ref; + if (flakeRef.rev) newRef.rev = flakeRef.rev; } + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); } - throw Error("cannot find flake with alias '%s' in the flake registry or in the flake lock file", refData->alias); - } else - return flakeRef; + } + if (!flakeRef.isDirect()) + throw Error("indirect flake URI '%s' is the result of a lookup", flakeRef.to_string()); + return flakeRef; } struct FlakeSourceInfo @@ -302,6 +231,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo { FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); + // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&fRef.data)) { if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string()); @@ -332,6 +262,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo return info; } + // This downloads the entire git history else if (auto refData = std::get_if(&fRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); @@ -342,7 +273,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo return info; } - else if (auto refData = std::get_if(&directFlakeRef.data)) { + else if (auto refData = std::get_if(&fRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, "", "source"); @@ -452,7 +383,7 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake = true) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake) { Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); Dependencies deps(flake); @@ -461,7 +392,7 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impur deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); for (auto & newFlakeRef : flake.requires) - deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, impureTopRef, false)); + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, false)); return deps; } @@ -505,25 +436,23 @@ void updateLockFile(EvalState & state, Path path) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) +Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) { - FlakeRef flakeRef = FlakeRef(flakeUri); + Dependencies deps = resolveFlake(state, flakeRef, impureTopRef); - Dependencies deps = resolveFlake(state, flakeRef, impure); - - // // FIXME: we should call each flake with only its dependencies - // // (rather than the closure of the top-level flake). + // FIXME: we should call each flake with only its dependencies + // (rather than the closure of the top-level flake). auto vResult = state.allocValue(); // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. state.mkAttrs(*vResult, deps.flakeDeps.size()); - Value * vTop = 0; + Value * vTop = state.allocAttr(*vResult, deps.flake.id); - for (auto & flake : deps.flakeDeps) { + for (auto & dep : deps.flakeDeps) { + Flake flake = dep.flake; auto vFlake = state.allocAttr(*vResult, flake.id); - if (deps.topFlakeId == flake.id) vTop = vFlake; state.mkAttrs(*vFlake, 4); @@ -532,7 +461,7 @@ Value * makeFlakeValue(EvalState & state, FlakeUri flakeUri, Value & v) state.store->assertStorePath(flake.path); mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path}); - if (flake.second.revCount) + if (flake.revCount) mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index adf8b07af..9da065234 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -10,13 +10,7 @@ class EvalState; struct FlakeRegistry { - struct Entry - { - FlakeRef ref; - Entry(const FlakeRef & flakeRef) : ref(flakeRef) {}; - Entry operator=(const Entry & entry) { return Entry(entry.ref); } - }; - std::map entries; + std::map entries; }; struct LockFile @@ -79,7 +73,7 @@ struct Dependencies Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake); +Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake = true); FlakeRegistry updateLockFile(EvalState &, Flake &); diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 32904953a..d789a6f70 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -103,48 +103,60 @@ typedef std::string FlakeUri; struct FlakeRef { - std::optional ref; - std::optional rev; - struct IsAlias { FlakeAlias alias; + bool operator<(const IsAlias & b) const { return alias < b.alias; }; + bool operator==(const IsAlias & b) const { return alias == b.alias; }; }; - struct IsGitHub - { + struct IsGitHub { std::string owner, repo; + bool operator<(const IsGitHub & b) const { + return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); + } + bool operator==(const IsGitHub & b) const { + return owner == b.owner && repo == b.repo; + } }; // Git, Tarball struct IsGit { std::string uri; + bool operator<(const IsGit & b) const { return uri < b.uri; } + bool operator==(const IsGit & b) const { return uri == b.uri; } }; struct IsPath { Path path; + bool operator<(const IsPath & b) const { return path < b.path; } + bool operator==(const IsPath & b) const { return path == b.path; } }; // Git, Tarball - std::variant data; + std::variant data; + + std::optional ref; + std::optional rev; + + bool operator<(const FlakeRef & flakeRef) const + { + return std::make_tuple(this->data, ref, rev) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + } + + bool operator==(const FlakeRef & flakeRef) const + { + return std::make_tuple(this->data, ref, rev) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + } // Parse a flake URI. FlakeRef(const std::string & uri, bool allowRelative = false); - // Default constructor - FlakeRef(const FlakeRef & flakeRef) : data(flakeRef.data) {}; - - /* Unify two flake references so that the resulting reference - combines the information from both. For example, - "nixpkgs/" and "github:NixOS/nixpkgs" unifies to - "nixpkgs/master". May throw an exception if the references are - incompatible (e.g. "nixpkgs/" and "nixpkgs/", - where hash1 != hash2). */ - FlakeRef(const FlakeRef & a, const FlakeRef & b); - // FIXME: change to operator <<. std::string to_string() const; @@ -160,9 +172,5 @@ struct FlakeRef bool isImmutable() const; FlakeRef baseRef() const; - - void setRef(std::optional ref) { ref = ref; } - - void setRev(std::optional rev) { rev = rev; } }; } diff --git a/src/nix/build.cc b/src/nix/build.cc index a2fc56e69..5a3d9d31a 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,4 +1,3 @@ -#include "primops/flake.hh" #include "eval.hh" #include "command.hh" #include "common-args.hh" @@ -78,7 +77,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - // std::string flakeUri = ""; + // FlakeUri flakeUri = ""; // if(updateLock) // for (uint i = 0; i < installables.size(); i++) // // if (auto flakeUri = installableToFlakeUri) diff --git a/src/nix/command.hh b/src/nix/command.hh index 83959bf9a..56e1e6f34 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,6 +1,7 @@ #pragma once #include "args.hh" +#include "primops/flake.hh" #include "common-eval-args.hh" namespace nix { @@ -46,7 +47,7 @@ struct GitRepoCommand : virtual Args struct FlakeCommand : virtual Args { - std::string flakeUri; + FlakeUri flakeUri; FlakeCommand() { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index df944a148..dbf0d3e9a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,4 +1,3 @@ -#include "primops/flake.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -29,11 +28,9 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs stopProgressBar(); - for (auto & registry : registries) { - for (auto & entry : registry->entries) { - std::cout << entry.first << " " << entry.second.ref.to_string() << "\n"; - } - } + for (auto & registry : registries) + for (auto & entry : registry->entries) + std::cout << entry.first.to_string() << " " << entry.second.to_string() << "\n"; } }; @@ -81,7 +78,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, true); std::queue todo; todo.push(deps); @@ -135,15 +132,15 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri)); + nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; struct CmdFlakeAdd : MixEvalArgs, Command { - FlakeAlias flakeAlias; - FlakeUri flakeUri; + FlakeUri alias; + FlakeUri uri; std::string name() override { @@ -157,25 +154,24 @@ struct CmdFlakeAdd : MixEvalArgs, Command CmdFlakeAdd() { - expectArg("flake-id", &flakeAlias); - expectArg("flake-uri", &flakeUri); + expectArg("alias", &alias); + expectArg("flake-uri", &uri); } void run() override { - FlakeRef newFlakeRef(flakeUri); + FlakeRef aliasRef(alias); Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - FlakeRegistry::Entry entry(newFlakeRef); - userRegistry->entries.erase(flakeAlias); - userRegistry->entries.insert_or_assign(flakeAlias, newFlakeRef); + userRegistry->entries.erase(aliasRef); + userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(uri)); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command { - FlakeAlias flakeAlias; + FlakeUri alias; std::string name() override { @@ -189,21 +185,21 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { - expectArg("flake-id", &flakeAlias); + expectArg("alias", &alias); } void run() override { Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); - userRegistry->entries.erase(flakeAlias); + userRegistry->entries.erase(FlakeRef(alias)); writeRegistry(*userRegistry, userRegistryPath); } }; struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs { - FlakeAlias flakeAlias; + FlakeUri alias; std::string name() override { @@ -217,7 +213,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { - expectArg("flake-id", &flakeAlias); + expectArg("alias", &alias); } void run(nix::ref store) override @@ -226,14 +222,13 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs Path userRegistryPath = getUserRegistryPath(); FlakeRegistry userRegistry = *readRegistry(userRegistryPath); - auto it = userRegistry.entries.find(flakeAlias); + auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - FlakeRef oldRef = it->second.ref; - it->second.ref = getFlake(*evalState, oldRef, true).ref; + it->second = getFlake(*evalState, it->second, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); } else - throw Error("the flake alias '%s' does not exist in the user registry", flakeAlias); + throw Error("the flake alias '%s' does not exist in the user registry", alias); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index e792ce96d..13a68a797 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,7 +7,6 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" -#include "primops/flake.hh" #include From 4bf3a8226badcdc70c013dfcfa266ee72f6cb89b Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0111/2548] Automated lockfile updating with `nix build` --- src/nix/build.cc | 16 ++++++++++------ src/nix/command.hh | 6 ++++++ src/nix/installables.cc | 8 ++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index da7c7f614..608946378 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,7 +11,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; - std::optional gitRepo = std::nullopt; + bool update = true; CmdBuild() { @@ -28,9 +28,9 @@ struct CmdBuild : MixDryRun, InstallablesCommand .set(&outLink, Path("")); mkFlag() - .longName("update-lock-file") - .description("update the lock file") - .dest(&gitRepo); + .longName("no-update") + .description("don't update the lock files") + .set(&update, false); } std::string name() override @@ -78,8 +78,12 @@ struct CmdBuild : MixDryRun, InstallablesCommand } } - if (gitRepo) - updateLockFile(*evalState, *gitRepo); + if (update) + for (auto installable : installables) { + auto flakeUri = installable->installableToFlakeUri(); + if (flakeUri) + updateLockFile(*evalState, *flakeUri); + } } }; diff --git a/src/nix/command.hh b/src/nix/command.hh index 83959bf9a..5d0c0c82c 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -2,6 +2,7 @@ #include "args.hh" #include "common-eval-args.hh" +#include namespace nix { @@ -65,6 +66,11 @@ struct Installable Buildable toBuildable(); + virtual std::optional installableToFlakeUri() + { + return std::nullopt; + } + virtual Value * toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index e792ce96d..43e15849b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -176,6 +176,14 @@ struct InstallableFlake : InstallableValue state.forceValue(*v); return v; } + + std::optional installableToFlakeUri() override + { + if (std::get_if(&flakeRef.data)) + return flakeRef.to_string(); + else + return std::nullopt; + } }; // FIXME: extend From bb6e6923f25841874b6a915d234d884ddd4c92dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Apr 2019 23:04:13 +0200 Subject: [PATCH 0112/2548] Add environment variable NIX_SHOW_SYMBOLS for dumping the symbol table --- src/libexpr/eval.cc | 6 ++++++ src/libexpr/symbol-table.hh | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 211f7a55f..d8e10d9f2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1811,6 +1811,7 @@ void EvalState::printStats() gc.attr("totalBytes", totalBytes); } #endif + if (countCalls) { { auto obj = topObj.object("primops"); @@ -1846,6 +1847,11 @@ void EvalState::printStats() } } } + + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 44929f7ee..91faea122 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -75,6 +75,13 @@ public: } size_t totalSize() const; + + template + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } }; } From 288f93cec0efeba5a767c09e4d58203c67704a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sun, 14 Apr 2019 03:18:33 +0200 Subject: [PATCH 0113/2548] manual: "Nix Package collection" -> "Nixpkgs package collection". Makes difference between Nix and Nixpkgs clearer to avoid some common confusion this sentence on IRC. Also disambiguate an "it" reference. --- doc/manual/packages/basic-package-mgmt.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/packages/basic-package-mgmt.xml b/doc/manual/packages/basic-package-mgmt.xml index e8d1419da..0f21297f3 100644 --- a/doc/manual/packages/basic-package-mgmt.xml +++ b/doc/manual/packages/basic-package-mgmt.xml @@ -24,11 +24,11 @@ symlinks to the files of the active applications. Components are installed from a set of Nix expressions that tell Nix how to build those packages, including, if necessary, their dependencies. There is a collection of -Nix expressions called the Nix Package collection that contains +Nix expressions called the Nixpkgs package collection that contains packages ranging from basic development stuff such as GCC and Glibc, to end-user applications like Mozilla Firefox. (Nix is however not -tied to the Nix Package collection; you could write your own Nix -expressions based on it, or completely new ones.) +tied to the Nixpkgs package collection; you could write your own Nix +expressions based on Nixpkgs, or completely new ones.) You can manually download the latest version of Nixpkgs from . However, From 84c12dbd7c8f2b34c46908f4a0c43cbb86023f20 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 13:45:51 +0200 Subject: [PATCH 0114/2548] Move --impure to MixEvalArgs --- src/libexpr/common-eval-args.cc | 7 +++++++ src/nix/installables.cc | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 37c74a94b..3e5400052 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -26,6 +26,13 @@ MixEvalArgs::MixEvalArgs() .description("add a path to the list of locations used to look up <...> file names") .label("path") .handler([&](std::string s) { searchPath.push_back(s); }); + + mkFlag() + .longName("impure") + .description("allow access to mutable paths and repositories") + .handler([&](std::vector ss) { + evalSettings.pureEval = false; + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index fc2c34861..9bc5ff41f 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -20,13 +20,6 @@ SourceExprCommand::SourceExprCommand() .label("file") .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); - - mkFlag() - .longName("impure") - .description("allow access to mutable paths and repositories") - .handler([&](std::vector ss) { - evalSettings.pureEval = false; - }); } ref SourceExprCommand::getEvalState() From f6d684b5e21966518d019c0225c3b0e8da2b6aff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 13:48:56 +0200 Subject: [PATCH 0115/2548] getFlakeRegistries(): Return registries regardless of pureEval This makes e.g. 'nix flake list' work. --- src/libexpr/primops/flake.cc | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 145d79446..e1eeffca6 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -126,7 +126,8 @@ void writeLockFile(LockFile lockFile, Path path) std::shared_ptr getGlobalRegistry() { - return std::make_shared(); + Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; + return readRegistry(registryFile); } Path getUserRegistryPath() @@ -139,12 +140,6 @@ std::shared_ptr getUserRegistry() return readRegistry(getUserRegistryPath()); } -std::shared_ptr getLocalRegistry() -{ - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); -} - std::shared_ptr getFlagRegistry() { // TODO (Nick): Implement this. @@ -154,15 +149,8 @@ std::shared_ptr getFlagRegistry() const std::vector> EvalState::getFlakeRegistries() { std::vector> registries; - if (evalSettings.pureEval) { - registries.push_back(std::make_shared()); // global - registries.push_back(std::make_shared()); // user - registries.push_back(std::make_shared()); // local - } else { - registries.push_back(getGlobalRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getLocalRegistry()); - } + registries.push_back(getGlobalRegistry()); + registries.push_back(getUserRegistry()); registries.push_back(getFlagRegistry()); return registries; } From c179f668e5bf64499169b17515bdc4c40473fca9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 14:08:18 +0200 Subject: [PATCH 0116/2548] Slight cleanup --- src/libexpr/primops/flake.cc | 7 +++---- src/libexpr/primops/flake.hh | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e1eeffca6..3c1cffba3 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -12,8 +12,7 @@ namespace nix { -/* Read the registry or a lock file. (Currently they have an identical - format. */ +/* Read a registry. */ std::shared_ptr readRegistry(const Path & path) { auto registry = std::make_shared(); @@ -34,8 +33,8 @@ std::shared_ptr readRegistry(const Path & path) return registry; } -/* Write the registry or lock file to a file. */ -void writeRegistry(FlakeRegistry registry, Path path) +/* Write a registry to a file. */ +void writeRegistry(const FlakeRegistry & registry, Path path) { nlohmann::json json; json["version"] = 1; diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 9da065234..8bfceb756 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -35,7 +35,7 @@ Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impure std::shared_ptr readRegistry(const Path &); -void writeRegistry(FlakeRegistry, Path); +void writeRegistry(const FlakeRegistry &, Path); struct Flake { From b4e367bf4a28b5495bc349df6fff0694ae73b9c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 14:13:10 +0200 Subject: [PATCH 0117/2548] FlakeRef::to_string(): Drop the "flake:" prefix This is unnecessary in most contexts and makes 'nix flake list' output less readable. --- src/libexpr/primops/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index ab1e5e152..274552218 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -120,7 +120,7 @@ std::string FlakeRef::to_string() const { std::string string; if (auto refData = std::get_if(&data)) - string = "flake:" + refData->alias; + string = refData->alias; else if (auto refData = std::get_if(&data)) { assert(!ref || !rev); From 5112a33fb17f792ceb6d641738277cbbe6a58bfc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 15 Apr 2019 19:17:17 +0200 Subject: [PATCH 0118/2548] Fix release script --- maintainers/upload-release.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 8432c9596..1cdf5ed16 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -67,10 +67,10 @@ sub downloadFile { } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt\n"; + print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } From b3d33b02e3fc40c7bd8f602334287825e7e6333d Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 10 Apr 2019 12:12:44 +0200 Subject: [PATCH 0119/2548] Added support for private github repositories --- src/libexpr/primops/flake.cc | 6 ++++-- src/libstore/globals.hh | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3c1cffba3..1b0b1eba7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -226,13 +226,15 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - // FIXME: support passing auth tokens for private repos. - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo, fRef.rev ? fRef.rev->to_string(Base16, false) : fRef.ref ? *fRef.ref : "master"); + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + auto result = getDownloader()->downloadCached(state.store, url, true, "source", Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..80d70fba3 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -344,6 +344,9 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting githubAccessToken{this, "", "github-acces-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; }; From 7587d62d02f216f28034f9e0938eb3236494c41b Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Tue, 16 Apr 2019 08:21:52 +0200 Subject: [PATCH 0120/2548] Fixed flake pin issues --- src/libexpr/primops/flake.hh | 2 ++ src/nix/flake.cc | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 8bfceb756..80114c7c8 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -63,6 +63,8 @@ struct NonFlake NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; }; +std::shared_ptr getGlobalRegistry(); + Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct Dependencies diff --git a/src/nix/flake.cc b/src/nix/flake.cc index dbf0d3e9a..8634733d6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -129,6 +129,8 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand return "list info about a given flake"; } + CmdFlakeInfo () { evalSettings.pureEval = false; } + void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); @@ -156,6 +158,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command { expectArg("alias", &alias); expectArg("flake-uri", &uri); + evalSettings.pureEval = false; } void run() override @@ -186,6 +189,7 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { expectArg("alias", &alias); + evalSettings.pureEval = false; } void run() override @@ -214,6 +218,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { expectArg("alias", &alias); + evalSettings.pureEval = false; } void run(nix::ref store) override @@ -227,8 +232,16 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs it->second = getFlake(*evalState, it->second, true).ref; // The 'ref' in 'flake' is immutable. writeRegistry(userRegistry, userRegistryPath); - } else - throw Error("the flake alias '%s' does not exist in the user registry", alias); + } else { + std::shared_ptr globalReg = getGlobalRegistry(); + it = globalReg->entries.find(FlakeRef(alias)); + if (it != globalReg->entries.end()) { + FlakeRef newRef = getFlake(*evalState, it->second, true).ref; + userRegistry.entries.insert_or_assign(alias, newRef); + writeRegistry(userRegistry, userRegistryPath); + } else + throw Error("the flake alias '%s' does not exist in the user or global registry", alias); + } } }; From d8fa2fc429c2dbaffce585e08d3070f912a29bf6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 12:26:17 +0200 Subject: [PATCH 0121/2548] Add FIXME for pureEval --- src/nix/flake.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8634733d6..3b37ad7a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -129,7 +129,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand return "list info about a given flake"; } - CmdFlakeInfo () { evalSettings.pureEval = false; } + CmdFlakeInfo () { evalSettings.pureEval = false; /* FIXME */ } void run(nix::ref store) override { @@ -158,7 +158,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command { expectArg("alias", &alias); expectArg("flake-uri", &uri); - evalSettings.pureEval = false; + evalSettings.pureEval = false; // FIXME } void run() override @@ -189,7 +189,7 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { expectArg("alias", &alias); - evalSettings.pureEval = false; + evalSettings.pureEval = false; // FIXME } void run() override @@ -218,7 +218,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { expectArg("alias", &alias); - evalSettings.pureEval = false; + evalSettings.pureEval = false; // FIXME } void run(nix::ref store) override From 529acfd24fdfb5e22eb3ec55b14e855ef845c98b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 12:36:10 +0200 Subject: [PATCH 0122/2548] Add nix to the flake registry --- flake-registry.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake-registry.json b/flake-registry.json index 378290ec6..422f77675 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -3,6 +3,9 @@ "dwarffs": { "uri": "github:edolstra/dwarffs/flake" }, + "nix": { + "uri": "github:tweag/nix/flakes" + }, "nixpkgs": { "uri": "github:edolstra/nixpkgs/flake" } From 035ac443544b46dc87274ed1eb1393b07db0912c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 13:56:08 +0200 Subject: [PATCH 0123/2548] Fix makeFlakeValue() --- src/libexpr/primops/flake.cc | 68 +++++++++++++++++------------------- src/libexpr/primops/flake.hh | 2 +- src/nix/installables.cc | 6 ++-- 3 files changed, 37 insertions(+), 39 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1b0b1eba7..bd8e5960c 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -304,6 +304,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.forceAttrs(vInfo); + // FIXME: change to "id"? if (auto name = vInfo.attrs->get(state.sName)) flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); else @@ -423,48 +424,45 @@ void updateLockFile(EvalState & state, Path path) } } -// Return the `provides` of the top flake, while assigning to `v` the provides -// of the dependencies as well. -Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) +void callFlake(EvalState & state, const Dependencies & flake, Value & v) { - Dependencies deps = resolveFlake(state, flakeRef, impureTopRef); + // Construct the resulting attrset '{description, provides, + // ...}'. This attrset is passed lazily as an argument to 'provides'. - // FIXME: we should call each flake with only its dependencies - // (rather than the closure of the top-level flake). + state.mkAttrs(v, flake.flakeDeps.size() + flake.nonFlakeDeps.size() + 4); - auto vResult = state.allocValue(); - // This will store the attribute set of the `nonFlakeRequires` and the `requires.provides`. - - state.mkAttrs(*vResult, deps.flakeDeps.size()); - - Value * vTop = state.allocAttr(*vResult, deps.flake.id); - - for (auto & dep : deps.flakeDeps) { - Flake flake = dep.flake; - auto vFlake = state.allocAttr(*vResult, flake.id); - - state.mkAttrs(*vFlake, 4); - - mkString(*state.allocAttr(*vFlake, state.sDescription), flake.description); - - state.store->assertStorePath(flake.path); - mkString(*state.allocAttr(*vFlake, state.sOutPath), flake.path, {flake.path}); - - if (flake.revCount) - mkInt(*state.allocAttr(*vFlake, state.symbols.create("revCount")), *flake.revCount); - - auto vProvides = state.allocAttr(*vFlake, state.symbols.create("provides")); - mkApp(*vProvides, *flake.vProvides, *vResult); - - vFlake->attrs->sort(); + for (auto & dep : flake.flakeDeps) { + auto vFlake = state.allocAttr(v, dep.flake.id); + callFlake(state, dep, *vFlake); } - vResult->attrs->sort(); + for (auto & dep : flake.nonFlakeDeps) { + auto vNonFlake = state.allocAttr(v, dep.alias); + state.mkAttrs(*vNonFlake, 4); - v = *vResult; + state.store->isValidPath(dep.path); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), dep.path, {dep.path}); + } - assert(vTop); - return vTop; + mkString(*state.allocAttr(v, state.sDescription), flake.flake.description); + + state.store->isValidPath(flake.flake.path); + mkString(*state.allocAttr(v, state.sOutPath), flake.flake.path, {flake.flake.path}); + + if (flake.flake.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *flake.flake.revCount); + + auto vProvides = state.allocAttr(v, state.symbols.create("provides")); + mkApp(*vProvides, *flake.flake.vProvides, v); + + v.attrs->sort(); +} + +// Return the `provides` of the top flake, while assigning to `v` the provides +// of the dependencies as well. +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) +{ + callFlake(state, resolveFlake(state, flakeRef, impureTopRef), v); } // This function is exposed to be used in nix files. diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 80114c7c8..4cd41352d 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -31,7 +31,7 @@ Path getUserRegistryPath(); Value * makeFlakeRegistryValue(EvalState & state); -Value * makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v); +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v); std::shared_ptr readRegistry(const Path &); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 9bc5ff41f..37217397a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -146,10 +146,10 @@ struct InstallableFlake : InstallableValue Value * toValue(EvalState & state) override { - auto vTemp = state.allocValue(); - auto vFlake = *makeFlakeValue(state, flakeRef, true, *vTemp); + auto vFlake = state.allocValue(); + makeFlakeValue(state, flakeRef, true, *vFlake); - auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; + auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; state.forceValue(*vProvides); From aecf07b1d6c21b8f402545912ae6c053d0f12a11 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 14:08:14 +0200 Subject: [PATCH 0124/2548] Remove dead function --- src/libexpr/primops/flake.cc | 26 -------------------------- src/libexpr/primops/flake.hh | 2 -- 2 files changed, 28 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index bd8e5960c..ba303fe4a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -154,32 +154,6 @@ const std::vector> EvalState::getFlakeRegistries( return registries; } -// Creates a Nix attribute set value listing all dependencies, so they can be used in `provides`. -Value * makeFlakeRegistryValue(EvalState & state) -{ - auto v = state.allocValue(); - - auto registries = state.getFlakeRegistries(); - - int size = 0; - for (auto registry : registries) - size += registry->entries.size(); - state.mkAttrs(*v, size); - - for (auto & registry : registries) { - for (auto & entry : registry->entries) { - auto vEntry = state.allocAttr(*v, entry.first.to_string()); - state.mkAttrs(*vEntry, 2); - mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.to_string()); - vEntry->attrs->sort(); - } - } - - v->attrs->sort(); - - return v; -} - static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, std::vector> registries, std::vector pastSearches = {}) { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 4cd41352d..73446c908 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -29,8 +29,6 @@ struct LockFile Path getUserRegistryPath(); -Value * makeFlakeRegistryValue(EvalState & state); - void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v); std::shared_ptr readRegistry(const Path &); From ba66455636f40264d44c7e1fb87e13653b22042a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 14:10:05 +0200 Subject: [PATCH 0125/2548] Improve incremental build --- src/nix/build.cc | 1 + src/nix/command.hh | 3 +-- src/nix/flake.cc | 2 ++ src/nix/installables.cc | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index ef6b48969..9ef07dcdb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -3,6 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "primops/flake.hh" using namespace nix; diff --git a/src/nix/command.hh b/src/nix/command.hh index a5ae56fb9..5d0c0c82c 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,7 +1,6 @@ #pragma once #include "args.hh" -#include "primops/flake.hh" #include "common-eval-args.hh" #include @@ -48,7 +47,7 @@ struct GitRepoCommand : virtual Args struct FlakeCommand : virtual Args { - FlakeUri flakeUri; + std::string flakeUri; FlakeCommand() { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3b37ad7a0..2079b1c27 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -3,6 +3,8 @@ #include "shared.hh" #include "progress-bar.hh" #include "eval.hh" +#include "primops/flake.hh" + #include #include diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 37217397a..963321336 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,6 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "primops/flake.hh" #include From ed9d725392827ee1516ca90ca891b2e7a66b2859 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 14:16:20 +0200 Subject: [PATCH 0126/2548] getFlake(): Use impureIsAllowed This fixes 'nix build nixpkgs:hello' without --impure. --- src/libexpr/primops/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index ba303fe4a..32874f87b 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -253,7 +253,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); + FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got flake source '%s' with revision %s", sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); From e1d73edb10ca38184c85b3124b4c59c6f04a0851 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 14:23:10 +0200 Subject: [PATCH 0127/2548] writeLockFile(): Emit empty objects rather than null --- src/libexpr/primops/flake.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 32874f87b..f65ae09ea 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -114,9 +114,10 @@ void writeLockFile(LockFile lockFile, Path path) { nlohmann::json json; json["version"] = 1; - json["nonFlakeRequires"]; + json["nonFlakeRequires"] = nlohmann::json::object(); for (auto & x : lockFile.nonFlakeEntries) json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + json["requires"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) json["requires"][x.first] = flakeEntryToJson(x.second); createDirs(dirOf(path)); From 7b312a8762988ff7a8e0f0890fcd2406cd89c1a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 14:27:54 +0200 Subject: [PATCH 0128/2548] Pass stuff by reference --- src/libexpr/primops/flake.cc | 17 +++++++++-------- src/libexpr/primops/flake.hh | 7 ++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f65ae09ea..3d11d9ec4 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -34,7 +34,7 @@ std::shared_ptr readRegistry(const Path & path) } /* Write a registry to a file. */ -void writeRegistry(const FlakeRegistry & registry, Path path) +void writeRegistry(const FlakeRegistry & registry, const Path & path) { nlohmann::json json; json["version"] = 1; @@ -99,7 +99,7 @@ LockFile readLockFile(const Path & path) return lockFile; } -nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) +nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) { nlohmann::json json; json["uri"] = entry.ref.to_string(); @@ -110,7 +110,7 @@ nlohmann::json flakeEntryToJson(LockFile::FlakeEntry & entry) return json; } -void writeLockFile(LockFile lockFile, Path path) +void writeLockFile(const LockFile & lockFile, const Path & path) { nlohmann::json json; json["version"] = 1; @@ -156,7 +156,8 @@ const std::vector> EvalState::getFlakeRegistries( } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, - std::vector> registries, std::vector pastSearches = {}) + const std::vector> & registries, + std::vector pastSearches = {}) { for (std::shared_ptr registry : registries) { auto i = registry->entries.find(flakeRef); @@ -362,14 +363,14 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impur return deps; } -LockFile::FlakeEntry dependenciesToFlakeEntry(Dependencies & deps) +LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps) { LockFile::FlakeEntry entry(deps.flake.ref); - for (Dependencies & deps : deps.flakeDeps) + for (auto & deps : deps.flakeDeps) entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); - for (NonFlake & nonFlake : deps.nonFlakeDeps) + for (auto & nonFlake : deps.nonFlakeDeps) entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); return entry; @@ -385,7 +386,7 @@ LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef) return lockFile; } -void updateLockFile(EvalState & state, Path path) +void updateLockFile(EvalState & state, const Path & path) { // 'path' is the path to the local flake repo. FlakeRef flakeRef = FlakeRef("file://" + path); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 73446c908..347dd2077 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -33,7 +33,7 @@ void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTop std::shared_ptr readRegistry(const Path &); -void writeRegistry(const FlakeRegistry &, Path); +void writeRegistry(const FlakeRegistry &, const Path &); struct Flake { @@ -75,7 +75,8 @@ struct Dependencies Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake = true); -FlakeRegistry updateLockFile(EvalState &, Flake &); +FlakeRegistry updateLockFile(EvalState &, const Flake &); + +void updateLockFile(EvalState &, const Path & path); -void updateLockFile(EvalState &, Path path); } From 60834492aea935b8043cdc8ccbc1270edebbc20a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 15:02:02 +0200 Subject: [PATCH 0129/2548] Update lock files from InstallableFlake::toValue() This ensures that the lock file is updated *before* evaluating it, and that it gets updated for any nix command, not just 'nix build'. Also, while computing the lock file, allow arbitrary registry lookups, not just at top-level. Also, improve some error messages slightly. --- src/libexpr/primops/flake.cc | 42 +++++++++++++++++++----------------- src/libexpr/primops/flake.hh | 8 +++---- src/nix/build.cc | 14 ------------ src/nix/command.hh | 7 ++---- src/nix/flake.cc | 2 +- src/nix/installables.cc | 20 +++++++++-------- 6 files changed, 40 insertions(+), 53 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3d11d9ec4..37dadd474 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -159,6 +159,9 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const std::vector> & registries, std::vector pastSearches = {}) { + if (registries.empty() && !flakeRef.isDirect()) + throw Error("indirect flake reference '%s' is not allowed", flakeRef.to_string()); + for (std::shared_ptr registry : registries) { auto i = registry->entries.find(flakeRef); if (i != registry->entries.end()) { @@ -178,8 +181,10 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return lookupFlake(state, newRef, registries, pastSearches); } } + if (!flakeRef.isDirect()) - throw Error("indirect flake URI '%s' is the result of a lookup", flakeRef.to_string()); + throw Error("could not resolve flake reference '%s'", flakeRef.to_string()); + return flakeRef; } @@ -192,7 +197,8 @@ struct FlakeSourceInfo static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) { - FlakeRef fRef = lookupFlake(state, flakeRef, state.getFlakeRegistries()); + FlakeRef fRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&fRef.data)) { @@ -349,16 +355,18 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, bool impureTopRef, bool isTopFlake) +Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, + RegistryAccess registryAccess, bool isTopFlake) { - Flake flake = getFlake(state, topRef, isTopFlake && impureTopRef); + Flake flake = getFlake(state, topRef, + registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake)); Dependencies deps(flake); for (auto & nonFlakeInfo : flake.nonFlakeRequires) deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); for (auto & newFlakeRef : flake.requires) - deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, false)); + deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, registryAccess, false)); return deps; } @@ -376,9 +384,9 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps) return entry; } -LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef) +static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) { - Dependencies deps = resolveFlake(evalState, flakeRef, true); + Dependencies deps = resolveFlake(evalState, flakeRef, AllowRegistry); LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); LockFile lockFile; lockFile.flakeEntries = entry.flakeEntries; @@ -388,16 +396,9 @@ LockFile getLockFile(EvalState & evalState, FlakeRef & flakeRef) void updateLockFile(EvalState & state, const Path & path) { - // 'path' is the path to the local flake repo. - FlakeRef flakeRef = FlakeRef("file://" + path); - if (std::get_if(&flakeRef.data)) { - LockFile lockFile = getLockFile(state, flakeRef); - writeLockFile(lockFile, path + "/flake.lock"); - } else if (std::get_if(&flakeRef.data)) { - throw UsageError("you can only update local flakes, not flakes on GitHub"); - } else { - throw UsageError("you can only update local flakes, not flakes through their FlakeAlias"); - } + FlakeRef flakeRef = FlakeRef("file://" + path); // FIXME: ugly + auto lockFile = makeLockFile(state, flakeRef); + writeLockFile(lockFile, path + "/flake.lock"); } void callFlake(EvalState & state, const Dependencies & flake, Value & v) @@ -436,15 +437,16 @@ void callFlake(EvalState & state, const Dependencies & flake, Value & v) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v) +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v) { - callFlake(state, resolveFlake(state, flakeRef, impureTopRef), v); + callFlake(state, resolveFlake(state, flakeRef, registryAccess), v); } // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), false, v); + makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 347dd2077..655d87f03 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -29,7 +29,9 @@ struct LockFile Path getUserRegistryPath(); -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, bool impureTopRef, Value & v); +enum RegistryAccess { DisallowRegistry, AllowRegistry, AllowRegistryAtTop }; + +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v); std::shared_ptr readRegistry(const Path &); @@ -73,9 +75,7 @@ struct Dependencies Dependencies(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, bool impureTopRef, bool isTopFlake = true); - -FlakeRegistry updateLockFile(EvalState &, const Flake &); +Dependencies resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); void updateLockFile(EvalState &, const Path & path); diff --git a/src/nix/build.cc b/src/nix/build.cc index 9ef07dcdb..d6a6a8071 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -11,8 +11,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand { Path outLink = "result"; - bool update = true; - CmdBuild() { mkFlag() @@ -26,11 +24,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand .longName("no-link") .description("do not create a symlink to the build result") .set(&outLink, Path("")); - - mkFlag() - .longName("no-update") - .description("don't update the lock file") - .set(&update, false); } std::string name() override @@ -77,13 +70,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand store2->addPermRoot(output.second, absPath(symlink), true); } } - - if (update) - for (auto installable : installables) { - auto flakeUri = installable->installableToFlakeUri(); - if (flakeUri) - updateLockFile(*evalState, *flakeUri); - } } }; diff --git a/src/nix/command.hh b/src/nix/command.hh index 5d0c0c82c..a52fbb9ba 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -66,11 +66,6 @@ struct Installable Buildable toBuildable(); - virtual std::optional installableToFlakeUri() - { - return std::nullopt; - } - virtual Value * toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); @@ -81,6 +76,8 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { std::optional file; + bool updateLockFile = true; + SourceExprCommand(); ref getEvalState(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2079b1c27..1e03669c3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -80,7 +80,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, true); + Dependencies deps = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop); std::queue todo; todo.push(deps); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 963321336..9d87c70c3 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -21,6 +21,11 @@ SourceExprCommand::SourceExprCommand() .label("file") .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); + + mkFlag() + .longName("no-update") + .description("don't create/update flake lock files") + .set(&updateLockFile, false); } ref SourceExprCommand::getEvalState() @@ -147,8 +152,13 @@ struct InstallableFlake : InstallableValue Value * toValue(EvalState & state) override { + auto path = std::get_if(&flakeRef.data); + if (cmd.updateLockFile && path) { + updateLockFile(state, path->path); + } + auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, true, *vFlake); + makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; @@ -169,14 +179,6 @@ struct InstallableFlake : InstallableValue state.forceValue(*v); return v; } - - std::optional installableToFlakeUri() override - { - if (std::get_if(&flakeRef.data)) - return flakeRef.to_string(); - else - return std::nullopt; - } }; // FIXME: extend From cfca793a20862220c53094ee63523c9a09d2c9a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 15:06:40 +0200 Subject: [PATCH 0130/2548] Remove unneeded pureEval flags --- src/nix/flake.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1e03669c3..93af71ac3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -131,7 +131,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand return "list info about a given flake"; } - CmdFlakeInfo () { evalSettings.pureEval = false; /* FIXME */ } + CmdFlakeInfo () { } void run(nix::ref store) override { @@ -160,7 +160,6 @@ struct CmdFlakeAdd : MixEvalArgs, Command { expectArg("alias", &alias); expectArg("flake-uri", &uri); - evalSettings.pureEval = false; // FIXME } void run() override @@ -191,7 +190,6 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command CmdFlakeRemove() { expectArg("alias", &alias); - evalSettings.pureEval = false; // FIXME } void run() override @@ -220,7 +218,6 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs CmdFlakePin() { expectArg("alias", &alias); - evalSettings.pureEval = false; // FIXME } void run(nix::ref store) override From 8c4e759efd779e80d135516fdce2cf884a2e06f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 15:11:17 +0200 Subject: [PATCH 0131/2548] updateLockFile(): Make sure Git can see flake.lock --- src/libexpr/primops/flake.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 37dadd474..13928d9d5 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -399,6 +399,10 @@ void updateLockFile(EvalState & state, const Path & path) FlakeRef flakeRef = FlakeRef("file://" + path); // FIXME: ugly auto lockFile = makeLockFile(state, flakeRef); writeLockFile(lockFile, path + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git. Otherwise, + // exportGit will fail to copy it to the Nix store. + runProgram("git", true, { "-C", path, "add", "flake.lock" }); } void callFlake(EvalState & state, const Dependencies & flake, Value & v) From 3d0e81051fca850fb7b46d6299a94566b8c1ab62 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 15:40:58 +0200 Subject: [PATCH 0132/2548] Fix lock file generation Before: "requires": { "nixpkgs": { "uri": "nixpkgs" } }, After: "requires": { "nixpkgs": { "uri": "github:edolstra/nixpkgs/f10e8a02eb7fa2b4a070f30cf87f4efcc7f3186d" } }, --- src/libexpr/primops/flake.cc | 53 ++++++++++++++++----------------- src/libexpr/primops/flake.hh | 18 +++++++---- src/libexpr/primops/flakeref.cc | 2 +- src/nix/flake.cc | 4 +-- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 13928d9d5..23406327e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -188,13 +188,6 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, return flakeRef; } -struct FlakeSourceInfo -{ - Path storePath; - std::optional rev; - std::optional revCount; -}; - static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) { FlakeRef fRef = lookupFlake(state, flakeRef, @@ -226,9 +219,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - FlakeSourceInfo info; + FlakeSourceInfo info(fRef); info.storePath = result.path; info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + info.flakeRef.rev = info.rev; + info.flakeRef.ref = {}; return info; } @@ -237,10 +232,12 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo else if (auto refData = std::get_if(&fRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); - FlakeSourceInfo info; + FlakeSourceInfo info(fRef); info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); info.revCount = gitInfo.revCount; + info.flakeRef.rev = info.rev; + // FIXME: ensure info.flakeRef.ref is set. return info; } @@ -248,10 +245,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, "", "source"); - FlakeSourceInfo info; + FlakeSourceInfo info(fRef); info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); info.revCount = gitInfo.revCount; + info.flakeRef.rev = info.rev; return info; } @@ -265,24 +263,21 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe debug("got flake source '%s' with revision %s", sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); - auto flakePath = sourceInfo.storePath; - state.store->assertStorePath(flakePath); + state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) - state.allowedPaths->insert(flakePath); + state.allowedPaths->insert(sourceInfo.storePath); - Flake flake(flakeRef); + Flake flake(flakeRef, std::move(sourceInfo)); if (std::get_if(&flakeRef.data)) { - if (sourceInfo.rev) + // FIXME: ehm? + if (flake.sourceInfo.rev) flake.ref = FlakeRef(flakeRef.baseRef().to_string() - + "/" + sourceInfo.rev->to_string(Base16, false)); + + "/" + flake.sourceInfo.rev->to_string(Base16, false)); } - flake.path = flakePath; - flake.revCount = sourceInfo.revCount; - Value vInfo; - state.evalFile(flakePath + "/flake.nix", vInfo); // FIXME: symlink attack + state.evalFile(sourceInfo.storePath + "/flake.nix", vInfo); // FIXME: symlink attack state.forceAttrs(vInfo); @@ -317,7 +312,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } else throw Error("flake lacks attribute 'provides'"); - const Path lockFile = flakePath + "/flake.lock"; // FIXME: symlink attack + Path lockFile = sourceInfo.storePath + "/flake.lock"; // FIXME: symlink attack flake.lockFile = readLockFile(lockFile); @@ -373,7 +368,7 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps) { - LockFile::FlakeEntry entry(deps.flake.ref); + LockFile::FlakeEntry entry(deps.flake.sourceInfo.flakeRef); for (auto & deps : deps.flakeDeps) entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); @@ -396,7 +391,10 @@ static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) void updateLockFile(EvalState & state, const Path & path) { - FlakeRef flakeRef = FlakeRef("file://" + path); // FIXME: ugly + // FIXME: don't copy 'path' to the store (especially since we + // dirty it immediately afterwards). + + FlakeRef flakeRef = FlakeRef(path); // FIXME: ugly auto lockFile = makeLockFile(state, flakeRef); writeLockFile(lockFile, path + "/flake.lock"); @@ -427,11 +425,12 @@ void callFlake(EvalState & state, const Dependencies & flake, Value & v) mkString(*state.allocAttr(v, state.sDescription), flake.flake.description); - state.store->isValidPath(flake.flake.path); - mkString(*state.allocAttr(v, state.sOutPath), flake.flake.path, {flake.flake.path}); + auto & path = flake.flake.sourceInfo.storePath; + state.store->isValidPath(path); + mkString(*state.allocAttr(v, state.sOutPath), path, {path}); - if (flake.flake.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *flake.flake.revCount); + if (flake.flake.sourceInfo.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *flake.flake.sourceInfo.revCount); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *flake.flake.vProvides, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 655d87f03..f93796660 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -37,20 +37,28 @@ std::shared_ptr readRegistry(const Path &); void writeRegistry(const FlakeRegistry &, const Path &); +struct FlakeSourceInfo +{ + FlakeRef flakeRef; + Path storePath; + std::optional rev; + std::optional revCount; + // date + FlakeSourceInfo(const FlakeRef & flakeRef) : flakeRef(flakeRef) { } +}; + struct Flake { FlakeId id; FlakeRef ref; std::string description; - Path path; - std::optional revCount; + FlakeSourceInfo sourceInfo; std::vector requires; LockFile lockFile; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc - // date - // content hash - Flake(const FlakeRef flakeRef) : ref(flakeRef) {}; + Flake(const FlakeRef & flakeRef, FlakeSourceInfo && sourceInfo) + : ref(flakeRef), sourceInfo(sourceInfo) {}; }; struct NonFlake diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 274552218..4127e63cc 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -123,7 +123,7 @@ std::string FlakeRef::to_string() const string = refData->alias; else if (auto refData = std::get_if(&data)) { - assert(!ref || !rev); + assert(!(ref && rev)); string = "github:" + refData->owner + "/" + refData->repo; } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 93af71ac3..7006ab989 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -40,13 +40,13 @@ void printFlakeInfo(Flake & flake, bool json) { if (json) { nlohmann::json j; j["id"] = flake.id; - j["location"] = flake.path; + j["location"] = flake.sourceInfo.storePath; j["description"] = flake.description; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; std::cout << "Description: " << flake.description << "\n"; - std::cout << "Location: " << flake.path << "\n"; + std::cout << "Location: " << flake.sourceInfo.storePath << "\n"; } } From 54ca4b4e81cd814d7727382bce073f6c6ea0ddf8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 15:47:15 +0200 Subject: [PATCH 0133/2548] Add flake lockfile --- flake.lock | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 flake.lock diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..3154006c8 --- /dev/null +++ b/flake.lock @@ -0,0 +1,9 @@ +{ + "nonFlakeRequires": {}, + "requires": { + "nixpkgs": { + "uri": "github:edolstra/nixpkgs/f10e8a02eb7fa2b4a070f30cf87f4efcc7f3186d" + } + }, + "version": 1 +} \ No newline at end of file From 3c28cb1b8ff586421ed9e37cef383af0486445cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 16:18:03 +0200 Subject: [PATCH 0134/2548] Improve 'nix flake info' a bit Example: $ nix flake info dwarffs ID: dwarffs URI: github:edolstra/dwarffs/a83d182fe3fe528ed6366a5cec3458bcb1a5f6e1 Description: A filesystem that fetches DWARF debug info from the Internet on demand Revision: a83d182fe3fe528ed6366a5cec3458bcb1a5f6e1 Path: /nix/store/grgd14kxxk8q4n503j87mpz48gcqpqw7-source --- src/nix/flake.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7006ab989..4b8f1026e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -40,13 +40,19 @@ void printFlakeInfo(Flake & flake, bool json) { if (json) { nlohmann::json j; j["id"] = flake.id; - j["location"] = flake.sourceInfo.storePath; + j["uri"] = flake.sourceInfo.flakeRef.to_string(); j["description"] = flake.description; + if (flake.sourceInfo.rev) + j["revision"] = flake.sourceInfo.rev->to_string(Base16, false); + j["path"] = flake.sourceInfo.storePath; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; + std::cout << "URI: " << flake.sourceInfo.flakeRef.to_string() << "\n"; std::cout << "Description: " << flake.description << "\n"; - std::cout << "Location: " << flake.sourceInfo.storePath << "\n"; + if (flake.sourceInfo.rev) + std::cout << "Revision: " << flake.sourceInfo.rev->to_string(Base16, false) << "\n"; + std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; } } From 260527a90ccc23461cdc4ad73970dd4e0b2e5239 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 16:18:47 +0200 Subject: [PATCH 0135/2548] Use the lock file --- src/libexpr/primops/flake.cc | 20 +++++++++++++------- src/libexpr/primops/flake.hh | 5 ++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 23406327e..193b521a3 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -106,7 +106,7 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) for (auto & x : entry.nonFlakeEntries) json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); for (auto & x : entry.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); + json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); return json; } @@ -119,7 +119,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path) json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); json["requires"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) - json["requires"][x.first] = flakeEntryToJson(x.second); + json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); createDirs(dirOf(path)); writeFile(path, json.dump(4)); // '4' = indentation in json file } @@ -312,10 +312,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } else throw Error("flake lacks attribute 'provides'"); - Path lockFile = sourceInfo.storePath + "/flake.lock"; // FIXME: symlink attack - - flake.lockFile = readLockFile(lockFile); - return flake; } @@ -355,13 +351,23 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, { Flake flake = getFlake(state, topRef, registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake)); + + LockFile lockFile; + + if (isTopFlake) + lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack + Dependencies deps(flake); for (auto & nonFlakeInfo : flake.nonFlakeRequires) deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); - for (auto & newFlakeRef : flake.requires) + for (auto newFlakeRef : flake.requires) { + auto i = lockFile.flakeEntries.find(newFlakeRef); + if (i != lockFile.flakeEntries.end()) newFlakeRef = i->second.ref; + // FIXME: propagate lockFile downwards deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, registryAccess, false)); + } return deps; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index f93796660..85f4fdf9f 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -18,12 +18,12 @@ struct LockFile struct FlakeEntry { FlakeRef ref; - std::map flakeEntries; + std::map flakeEntries; std::map nonFlakeEntries; FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; }; - std::map flakeEntries; + std::map flakeEntries; std::map nonFlakeEntries; }; @@ -54,7 +54,6 @@ struct Flake std::string description; FlakeSourceInfo sourceInfo; std::vector requires; - LockFile lockFile; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc Flake(const FlakeRef & flakeRef, FlakeSourceInfo && sourceInfo) From 3ddb6d1833a94e3d141116f3e579e66ebef04111 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 16:24:51 +0200 Subject: [PATCH 0136/2548] Allow refs to start with a digit E.g. we want to accept "19.03" as a ref. --- src/libexpr/primops/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 4127e63cc..973987469 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -5,7 +5,7 @@ namespace nix { // A Git ref (i.e. branch or tag name). -const static std::string refRegex = "[a-zA-Z][a-zA-Z0-9_.-]*"; // FIXME: check +const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check // A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; From 939bee06cd7c68af1508fab127202689fc63c22e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Apr 2019 16:29:44 +0200 Subject: [PATCH 0137/2548] Pass a flake to itself as "self" --- flake.nix | 10 ++++++---- src/libexpr/primops/flake.cc | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 695f67fa4..95ec5d952 100644 --- a/flake.nix +++ b/flake.nix @@ -3,13 +3,15 @@ description = "The purely functional package manager"; - requires = [ flake:nixpkgs ]; + epoch = 2019; - provides = flakes: rec { + requires = [ "nixpkgs" ]; + + provides = deps: rec { hydraJobs = import ./release.nix { - nix = flakes.nix; # => flakes.self? - nixpkgs = flakes.nixpkgs; + nix = deps.self; + nixpkgs = deps.nixpkgs; }; packages.nix = hydraJobs.build.x86_64-linux; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 193b521a3..296db3f92 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -441,6 +441,8 @@ void callFlake(EvalState & state, const Dependencies & flake, Value & v) auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *flake.flake.vProvides, v); + v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + v.attrs->sort(); } From b42ba08fc8a291c549c1f9f92457d72639fac995 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0138/2548] Add command `flake clone` --- src/libexpr/primops/flake.cc | 43 +++++++++++++++++++++++++++++---- src/libexpr/primops/flake.hh | 3 +++ src/libexpr/primops/flakeref.hh | 1 + src/libutil/util.cc | 7 +++++- src/nix/flake.cc | 29 ++++++++++++++++++++++ 5 files changed, 77 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 296db3f92..0e4b8afee 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -146,17 +146,19 @@ std::shared_ptr getFlagRegistry() return std::make_shared(); } -const std::vector> EvalState::getFlakeRegistries() +// This always returns a vector with globalReg, userReg, localReg, flakeReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() { - std::vector> registries; - registries.push_back(getGlobalRegistry()); + Registries registries; + registries.push_back(getGlobalRegistry()); // TODO (Nick): Doesn't this break immutability? registries.push_back(getUserRegistry()); + registries.push_back(std::make_shared()); // local registries.push_back(getFlagRegistry()); return registries; } -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, - const std::vector> & registries, +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, std::vector pastSearches = {}) { if (registries.empty() && !flakeRef.isDirect()) @@ -462,4 +464,35 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va static RegisterPrimOp r2("getFlake", 1, prim_getFlake); +void gitCloneFlake (std::string flakeUri, EvalState & state, Registries registries, + Path endDirectory) +{ + FlakeRef flakeRef(flakeUri); + flakeRef = lookupFlake(state, flakeRef, registries); + + std::string uri; + + Strings args = {"clone"}; + + if (auto refData = std::get_if(&flakeRef.data)) { + uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; + args.push_back(uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } else if (auto refData = std::get_if(&flakeRef.data)) { + args.push_back(refData->uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } + + if (endDirectory != "") + args.push_back(endDirectory); + + runProgram("git", true, args); +} + } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 85f4fdf9f..76219fbd6 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -27,6 +27,8 @@ struct LockFile std::map nonFlakeEntries; }; +typedef std::vector> Registries; + Path getUserRegistryPath(); enum RegistryAccess { DisallowRegistry, AllowRegistry, AllowRegistryAtTop }; @@ -86,4 +88,5 @@ Dependencies resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registry void updateLockFile(EvalState &, const Path & path); +void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path); } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index d789a6f70..cf9d7a1a6 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -67,6 +67,7 @@ namespace nix { https://example.org/my/repo.git https://example.org/my/repo.git?ref=release-1.2.3 https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + git://github.com/edolstra/dwarffs.git\?ref=flake\&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 * /path.git(\?attr(&attr)*)? diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b0a2b853e..f4f86c5c8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -962,12 +962,14 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } - +// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional & input) { RunOptions opts(program, args); opts.searchPath = searchPath; + // This allows you to refer to a program with a pathname relative to the + // PATH variable. opts.input = input; auto res = runProgram(opts); @@ -978,6 +980,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } +// Output = error code + "standard out" output stream std::pair runProgram(const RunOptions & options_) { RunOptions options(options_); @@ -1028,6 +1031,8 @@ void runProgram2(const RunOptions & options) if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4b8f1026e..35324295d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -280,6 +280,34 @@ struct CmdFlakeInit : virtual Args, Command } }; +struct CmdFlakeClone : StoreCommand, FlakeCommand, MixEvalArgs +{ + Path endDirectory = ""; + + std::string name() override + { + return "clone"; + } + + std::string description() override + { + return "clone flake repository"; + } + + CmdFlakeClone() + { + expectArg("end-dir", &endDirectory, true); + } + + void run(nix::ref store) override + { + auto evalState = std::make_shared(searchPath, store); + + Registries registries = evalState->getFlakeRegistries(); + gitCloneFlake(flakeUri, *evalState, registries, endDirectory); + } +}; + struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() @@ -291,6 +319,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command , make_ref() , make_ref() , make_ref() + , make_ref() }) { } From 160b974fb0623df436c9e834f6d4db62dfda02d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Apr 2019 13:54:06 +0200 Subject: [PATCH 0139/2548] Fix mutability check --- src/libexpr/primops/flake.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 296db3f92..0bf666a98 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -193,10 +193,11 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo FlakeRef fRef = lookupFlake(state, flakeRef, impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", fRef.to_string()); + // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&fRef.data)) { - if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", fRef.to_string()); // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. From b614e0e53da1f825e198996f88935a1d1a89725f Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Wed, 17 Apr 2019 21:42:41 +0200 Subject: [PATCH 0140/2548] build: only skip hash rewriting for sandboxing on linux The sandbox on darwin, and possibly other future platforms, doesn't have enough isolation to redirect outputs without hash rewriting. --- src/libstore/build.cc | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0c743b..dbadfacd5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -803,6 +803,9 @@ private: /* Whether we're currently doing a chroot build. */ bool useChroot = false; + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite; + Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -994,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , wantedOutputs(wantedOutputs) , buildMode(buildMode) { +#if __linux__ + needsHashRewrite = !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + needsHashRewrite = true; +#endif + state = &DerivationGoal::getDerivation; name = (format("building of '%1%'") % drvPath).str(); trace("created"); @@ -2073,7 +2083,7 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -2873,6 +2883,10 @@ void DerivationGoal::runChild() for (auto & i : missingPaths) { sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -3051,7 +3065,9 @@ void DerivationGoal::registerOutputs() throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { + } + + if (needsHashRewrite) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() From 6e4210d8ce76f52d9fd717660ea24b98ba780843 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 10:58:08 +0200 Subject: [PATCH 0141/2548] Fix assertion failure --- src/libexpr/primops/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 0bf666a98..2415faf5e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -415,7 +415,7 @@ void callFlake(EvalState & state, const Dependencies & flake, Value & v) // Construct the resulting attrset '{description, provides, // ...}'. This attrset is passed lazily as an argument to 'provides'. - state.mkAttrs(v, flake.flakeDeps.size() + flake.nonFlakeDeps.size() + 4); + state.mkAttrs(v, flake.flakeDeps.size() + flake.nonFlakeDeps.size() + 8); for (auto & dep : flake.flakeDeps) { auto vFlake = state.allocAttr(v, dep.flake.id); From 46cb15df9b3501ca631779fa7d5c6299c1c17b53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 11:16:14 +0200 Subject: [PATCH 0142/2548] Fix assertion failure in FlakeRef::to_string() --- src/libexpr/primops/fetchGit.cc | 3 ++- src/libexpr/primops/fetchGit.hh | 1 + src/libexpr/primops/flake.cc | 2 +- src/libexpr/primops/flakeref.cc | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 391308224..40975d8d8 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -39,6 +39,7 @@ GitInfo exportGit(ref store, const std::string & uri, files. */ GitInfo gitInfo; + gitInfo.ref = "HEAD"; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -67,7 +68,6 @@ GitInfo exportGit(ref store, const std::string & uri, // clean working tree, but no ref or rev specified. Use 'HEAD'. rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; } if (!ref) ref = "HEAD"s; @@ -127,6 +127,7 @@ GitInfo exportGit(ref store, const std::string & uri, // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; + gitInfo.ref = *ref; gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 60c439426..5937bdcc0 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -9,6 +9,7 @@ namespace nix { struct GitInfo { Path storePath; + std::string ref; std::string rev; std::string shortRev; std::optional revCount; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 2415faf5e..1149efaac 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -237,8 +237,8 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo info.storePath = gitInfo.storePath; info.rev = Hash(gitInfo.rev, htSHA1); info.revCount = gitInfo.revCount; + info.flakeRef.ref = gitInfo.ref; info.flakeRef.rev = info.rev; - // FIXME: ensure info.flakeRef.ref is set. return info; } diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 973987469..97f31377a 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -128,7 +128,7 @@ std::string FlakeRef::to_string() const } else if (auto refData = std::get_if(&data)) { - assert(ref || !rev); + assert(!rev || ref); string = refData->uri; } From 6960ee929dcf95c24e0db761fd4bc46c3749abb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 11:34:23 +0200 Subject: [PATCH 0143/2548] Clean up exportGit argument handling --- src/libexpr/primops/fetchGit.cc | 42 ++++++++++++++++----------------- src/libexpr/primops/fetchGit.hh | 6 ++--- src/libexpr/primops/flake.cc | 10 ++++---- src/libutil/hash.hh | 12 ++++++++++ 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 40975d8d8..3a6830cb7 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -19,10 +19,13 @@ namespace nix { extern std::regex revRegex; GitInfo exportGit(ref store, const std::string & uri, - std::optional ref, std::string rev, + std::optional ref, + std::optional rev, const std::string & name) { - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + assert(!rev || rev->type == htSHA1); + + if (!ref && !rev && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { bool clean = true; @@ -40,8 +43,6 @@ GitInfo exportGit(ref store, const std::string & uri, GitInfo gitInfo; gitInfo.ref = "HEAD"; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); auto files = tokenizeString>( runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); @@ -67,14 +68,11 @@ GitInfo exportGit(ref store, const std::string & uri, } // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); + rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })), htSHA1); } if (!ref) ref = "HEAD"s; - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - deletePath(getCacheDir() + "/nix/git"); Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); @@ -90,9 +88,9 @@ GitInfo exportGit(ref store, const std::string & uri, time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (rev != "") { + if (rev) { try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); + runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -128,19 +126,19 @@ GitInfo exportGit(ref store, const std::string & uri, // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; gitInfo.ref = *ref; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + gitInfo.rev = rev ? *rev : Hash(chomp(readFile(localRefFile)), htSHA1); printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, + name + std::string("\0"s) + gitInfo.rev.gitRev()).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken try { auto json = nlohmann::json::parse(readFile(storeLink)); - assert(json["name"] == name && json["rev"] == gitInfo.rev); + assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); gitInfo.storePath = json["storePath"]; @@ -155,7 +153,7 @@ GitInfo exportGit(ref store, const std::string & uri, // FIXME: should pipe this, or find some better way to extract a // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev.gitRev() }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -164,13 +162,13 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.storePath = store->addToStore(name, tmpDir); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; json["uri"] = uri; json["name"] = name; - json["rev"] = gitInfo.rev; + json["rev"] = gitInfo.rev.gitRev(); json["revCount"] = *gitInfo.revCount; writeFile(storeLink, json.dump()); @@ -182,7 +180,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va { std::string url; std::optional ref; - std::string rev; + std::optional rev; std::string name = "source"; PathSet context; @@ -199,7 +197,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -216,15 +214,15 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && rev == "") + if (evalSettings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); auto gitInfo = exportGit(state.store, url, ref, rev, name); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); + mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.rev.gitShortRev()); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount.value_or(0)); v.attrs->sort(); diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 5937bdcc0..a867f38f6 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -10,13 +10,13 @@ struct GitInfo { Path storePath; std::string ref; - std::string rev; - std::string shortRev; + Hash rev{htSHA1}; std::optional revCount; }; GitInfo exportGit(ref store, const std::string & uri, - std::optional ref, std::string rev, + std::optional ref, + std::optional rev, const std::string & name); } diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1149efaac..c5e646412 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -231,11 +231,10 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo // This downloads the entire git history else if (auto refData = std::get_if(&fRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, - fRef.rev ? fRef.rev->to_string(Base16, false) : "", "source"); + auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev, "source"); FlakeSourceInfo info(fRef); info.storePath = gitInfo.storePath; - info.rev = Hash(gitInfo.rev, htSHA1); + info.rev = gitInfo.rev; info.revCount = gitInfo.revCount; info.flakeRef.ref = gitInfo.ref; info.flakeRef.rev = info.rev; @@ -245,11 +244,12 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo else if (auto refData = std::get_if(&fRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); - auto gitInfo = exportGit(state.store, refData->path, {}, "", "source"); + auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); FlakeSourceInfo info(fRef); info.storePath = gitInfo.storePath; - info.rev = Hash(gitInfo.rev, htSHA1); + info.rev = gitInfo.rev; info.revCount = gitInfo.revCount; + info.flakeRef.ref = gitInfo.ref; info.flakeRef.rev = info.rev; return info; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2dbc3b630..edede8ace 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -80,6 +80,18 @@ struct Hash or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ std::string to_string(Base base = Base32, bool includeType = true) const; + + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } + + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } }; From 160ce18a0e9f569f94e6b0cb8e47bd4008a9fea2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 11:43:56 +0200 Subject: [PATCH 0144/2548] Improve missing flake.nix error message --- src/libexpr/primops/flake.cc | 18 +++++++++++------- src/libexpr/primops/flakeref.cc | 6 ++++++ src/libexpr/primops/flakeref.hh | 5 +++++ src/nix/flake.cc | 4 ++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c5e646412..720e157c6 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -48,7 +48,7 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) { FlakeRef flakeRef(json["uri"]); if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); LockFile::FlakeEntry entry(flakeRef); @@ -57,7 +57,7 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); } @@ -87,7 +87,7 @@ LockFile readLockFile(const Path & path) for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef.to_string()); + throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); } @@ -160,7 +160,7 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, std::vector pastSearches = {}) { if (registries.empty() && !flakeRef.isDirect()) - throw Error("indirect flake reference '%s' is not allowed", flakeRef.to_string()); + throw Error("indirect flake reference '%s' is not allowed", flakeRef); for (std::shared_ptr registry : registries) { auto i = registry->entries.find(flakeRef); @@ -183,7 +183,7 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, } if (!flakeRef.isDirect()) - throw Error("could not resolve flake reference '%s'", flakeRef.to_string()); + throw Error("could not resolve flake reference '%s'", flakeRef); return flakeRef; } @@ -194,7 +194,7 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) - throw Error("requested to fetch mutable flake '%s' in pure mode", fRef.to_string()); + throw Error("requested to fetch mutable flake '%s' in pure mode", fRef); // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&fRef.data)) { @@ -277,8 +277,12 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe + "/" + flake.sourceInfo.rev->to_string(Base16, false)); } + Path flakeFile = sourceInfo.storePath + "/flake.nix"; + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", flakeRef); + Value vInfo; - state.evalFile(sourceInfo.storePath + "/flake.nix", vInfo); // FIXME: symlink attack + state.evalFile(flakeFile, vInfo); // FIXME: symlink attack state.forceAttrs(vInfo); diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 97f31377a..b91bbee2a 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -142,6 +142,12 @@ std::string FlakeRef::to_string() const return string; } +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + bool FlakeRef::isImmutable() const { return (bool) rev; diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index d789a6f70..e599e2feb 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -1,3 +1,5 @@ +#pragma once + #include "types.hh" #include "hash.hh" @@ -173,4 +175,7 @@ struct FlakeRef FlakeRef baseRef() const; }; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4b8f1026e..34d67ee58 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -32,7 +32,7 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs for (auto & registry : registries) for (auto & entry : registry->entries) - std::cout << entry.first.to_string() << " " << entry.second.to_string() << "\n"; + std::cout << entry.first << " " << entry.second << "\n"; } }; @@ -48,7 +48,7 @@ void printFlakeInfo(Flake & flake, bool json) { std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; - std::cout << "URI: " << flake.sourceInfo.flakeRef.to_string() << "\n"; + std::cout << "URI: " << flake.sourceInfo.flakeRef << "\n"; std::cout << "Description: " << flake.description << "\n"; if (flake.sourceInfo.rev) std::cout << "Revision: " << flake.sourceInfo.rev->to_string(Base16, false) << "\n"; From 0cbda84f5b14aba0416cb65f88f8e9d487895207 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:06:27 +0200 Subject: [PATCH 0145/2548] exportGit: Don't clone local repositories This ensures that commands like 'nix flake info /my/nixpkgs' don't copy a gigabyte of crap to ~/.cache/nix. Fixes #60. --- src/libexpr/primops/fetchGit.cc | 129 +++++++++++++++--------- src/libexpr/primops/fetchGit.hh | 2 +- src/libstore/http-binary-cache-store.cc | 5 +- tests/binary-cache.sh | 4 +- tests/fetchGit.sh | 2 + 5 files changed, 88 insertions(+), 54 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 3a6830cb7..eb95208de 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -18,14 +18,19 @@ namespace nix { extern std::regex revRegex; -GitInfo exportGit(ref store, const std::string & uri, +GitInfo exportGit(ref store, std::string uri, std::optional ref, std::optional rev, const std::string & name) { assert(!rev || rev->type == htSHA1); - if (!ref && !rev && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git"); + + // If this is a local directory (but not a file:// URI) and no ref + // or revision is given, then allow the use of an unclean working + // tree. + if (!ref && !rev && isLocal) { bool clean = true; @@ -66,67 +71,92 @@ GitInfo exportGit(ref store, const std::string & uri, return gitInfo; } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })), htSHA1); } - if (!ref) ref = "HEAD"s; + if (!ref) ref = isLocal ? "HEAD" : "master"; + + // Don't clone file:// URIs (but otherwise treat them the same as + // remote URIs, i.e. don't use the working tree or HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + if (!forceHttp && hasPrefix(uri, "file://")) { + uri = std::string(uri, 7); + isLocal = true; + } deletePath(getCacheDir() + "/nix/git"); Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + Path repoDir; - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } + if (isLocal) { - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + if (!rev) + rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1); + + if (!pathExists(cacheDir)) + createDirs(cacheDir); + + repoDir = uri; - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev) { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + repoDir = cacheDir; - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } - utimes(localRefFile.c_str(), times); + Path localRefFile = repoDir + "/refs/heads/" + *ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + st.st_mtime + settings.tarballTtl <= now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!rev) + rev = Hash(chomp(readFile(localRefFile)), htSHA1); } // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; gitInfo.ref = *ref; - gitInfo.rev = rev ? *rev : Hash(chomp(readFile(localRefFile)), htSHA1); + gitInfo.rev = *rev; printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); @@ -140,9 +170,10 @@ GitInfo exportGit(ref store, const std::string & uri, assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); - gitInfo.storePath = json["storePath"]; + Path storePath = json["storePath"]; - if (store->isValidPath(gitInfo.storePath)) { + if (store->isValidPath(storePath)) { + gitInfo.storePath = storePath; gitInfo.revCount = json["revCount"]; return gitInfo; } @@ -153,7 +184,7 @@ GitInfo exportGit(ref store, const std::string & uri, // FIXME: should pipe this, or find some better way to extract a // revision. - auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev.gitRev() }); + auto tar = runProgram("git", true, { "-C", repoDir, "archive", gitInfo.rev.gitRev() }); Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); @@ -162,7 +193,7 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.storePath = store->addToStore(name, tmpDir); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev.gitRev() })); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index a867f38f6..32e748f98 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -14,7 +14,7 @@ struct GitInfo std::optional revCount; }; -GitInfo exportGit(ref store, const std::string & uri, +GitInfo exportGit(ref store, std::string uri, std::optional ref, std::optional rev, const std::string & name); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..105e1dcdd 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -160,10 +160,11 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared(params, uri); store->init(); return store; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..a3c3c7847 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -48,7 +48,7 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests @@ -126,7 +126,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 51fd49e9f..d87ce8560 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,6 +9,8 @@ clearStore repo=$TEST_ROOT/git +export _NIX_FORCE_HTTP=1 + rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 git init $repo From bc259192b4e1f90c575ddc83814b82cca829a4f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:15:51 +0200 Subject: [PATCH 0146/2548] fetchGit: Return revCount for dirty working trees --- src/libexpr/primops/fetchGit.cc | 5 +++-- src/libexpr/primops/fetchGit.hh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index eb95208de..e79eacafe 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -68,6 +68,7 @@ GitInfo exportGit(ref store, std::string uri, }; gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); return gitInfo; } @@ -200,7 +201,7 @@ GitInfo exportGit(ref store, std::string uri, json["uri"] = uri; json["name"] = name; json["rev"] = gitInfo.rev.gitRev(); - json["revCount"] = *gitInfo.revCount; + json["revCount"] = gitInfo.revCount; writeFile(storeLink, json.dump()); @@ -254,7 +255,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.rev.gitShortRev()); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount.value_or(0)); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 32e748f98..2ad6a5e5c 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -11,7 +11,7 @@ struct GitInfo Path storePath; std::string ref; Hash rev{htSHA1}; - std::optional revCount; + uint64_t revCount; }; GitInfo exportGit(ref store, std::string uri, From 50ec2bed9edd234eabbd4a3920052ca2f94bca52 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:19:46 +0200 Subject: [PATCH 0147/2548] nix flake info: Show revcount --- src/nix/flake.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 34d67ee58..eec280584 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -44,6 +44,8 @@ void printFlakeInfo(Flake & flake, bool json) { j["description"] = flake.description; if (flake.sourceInfo.rev) j["revision"] = flake.sourceInfo.rev->to_string(Base16, false); + if (flake.sourceInfo.revCount) + j["revCount"] = *flake.sourceInfo.revCount; j["path"] = flake.sourceInfo.storePath; std::cout << j.dump(4) << std::endl; } else { @@ -52,6 +54,8 @@ void printFlakeInfo(Flake & flake, bool json) { std::cout << "Description: " << flake.description << "\n"; if (flake.sourceInfo.rev) std::cout << "Revision: " << flake.sourceInfo.rev->to_string(Base16, false) << "\n"; + if (flake.sourceInfo.revCount) + std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; } } From e51abb6631ff0f5fc52523ea1819333cb587170c Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Fri, 19 Apr 2019 14:23:35 +0200 Subject: [PATCH 0148/2548] Changed some names --- src/libexpr/primops/flake.cc | 46 ++++++++++++++++++------------------ src/libexpr/primops/flake.hh | 8 +++---- src/nix/flake.cc | 14 +++++------ 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 720e157c6..c098168de 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -351,7 +351,7 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al dependencies. FIXME: this should return a graph of flakes. */ -Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, RegistryAccess registryAccess, bool isTopFlake) { Flake flake = getFlake(state, topRef, @@ -362,7 +362,7 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, if (isTopFlake) lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack - Dependencies deps(flake); + ResolvedFlake deps(flake); for (auto & nonFlakeInfo : flake.nonFlakeRequires) deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); @@ -377,14 +377,14 @@ Dependencies resolveFlake(EvalState & state, const FlakeRef & topRef, return deps; } -LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps) +LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - LockFile::FlakeEntry entry(deps.flake.sourceInfo.flakeRef); + LockFile::FlakeEntry entry(resolvedFlake.flake.sourceInfo.flakeRef); - for (auto & deps : deps.flakeDeps) - entry.flakeEntries.insert_or_assign(deps.flake.id, dependenciesToFlakeEntry(deps)); + for (auto & newResFlake : resolvedFlake.flakeDeps) + entry.flakeEntries.insert_or_assign(newResFlake.flake.id, dependenciesToFlakeEntry(newResFlake)); - for (auto & nonFlake : deps.nonFlakeDeps) + for (auto & nonFlake : resolvedFlake.nonFlakeDeps) entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); return entry; @@ -392,8 +392,8 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(const Dependencies & deps) static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) { - Dependencies deps = resolveFlake(evalState, flakeRef, AllowRegistry); - LockFile::FlakeEntry entry = dependenciesToFlakeEntry(deps); + ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, AllowRegistry); + LockFile::FlakeEntry entry = dependenciesToFlakeEntry(resFlake); LockFile lockFile; lockFile.flakeEntries = entry.flakeEntries; lockFile.nonFlakeEntries = entry.nonFlakeEntries; @@ -414,37 +414,37 @@ void updateLockFile(EvalState & state, const Path & path) runProgram("git", true, { "-C", path, "add", "flake.lock" }); } -void callFlake(EvalState & state, const Dependencies & flake, Value & v) +void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) { // Construct the resulting attrset '{description, provides, // ...}'. This attrset is passed lazily as an argument to 'provides'. - state.mkAttrs(v, flake.flakeDeps.size() + flake.nonFlakeDeps.size() + 8); + state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); - for (auto & dep : flake.flakeDeps) { - auto vFlake = state.allocAttr(v, dep.flake.id); - callFlake(state, dep, *vFlake); + for (const ResolvedFlake newResFlake : resFlake.flakeDeps) { + auto vFlake = state.allocAttr(v, newResFlake.flake.id); + callFlake(state, newResFlake, *vFlake); } - for (auto & dep : flake.nonFlakeDeps) { - auto vNonFlake = state.allocAttr(v, dep.alias); + for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { + auto vNonFlake = state.allocAttr(v, nonFlake.alias); state.mkAttrs(*vNonFlake, 4); - state.store->isValidPath(dep.path); - mkString(*state.allocAttr(*vNonFlake, state.sOutPath), dep.path, {dep.path}); + state.store->isValidPath(nonFlake.path); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.path, {nonFlake.path}); } - mkString(*state.allocAttr(v, state.sDescription), flake.flake.description); + mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); - auto & path = flake.flake.sourceInfo.storePath; + auto & path = resFlake.flake.sourceInfo.storePath; state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); - if (flake.flake.sourceInfo.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *flake.flake.sourceInfo.revCount); + if (resFlake.flake.sourceInfo.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); - mkApp(*vProvides, *flake.flake.vProvides, v); + mkApp(*vProvides, *resFlake.flake.vProvides, v); v.attrs->push_back(Attr(state.symbols.create("self"), &v)); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 85f4fdf9f..0b70088cc 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -74,15 +74,15 @@ std::shared_ptr getGlobalRegistry(); Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); -struct Dependencies +struct ResolvedFlake { Flake flake; - std::vector flakeDeps; // The flake dependencies + std::vector flakeDeps; // The flake dependencies std::vector nonFlakeDeps; - Dependencies(const Flake & flake) : flake(flake) {} + ResolvedFlake(const Flake & flake) : flake(flake) {} }; -Dependencies resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); void updateLockFile(EvalState &, const Path & path); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 34d67ee58..07e9e313a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -86,20 +86,20 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - Dependencies deps = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop); + ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop); - std::queue todo; - todo.push(deps); + std::queue todo; + todo.push(resFlake); while (!todo.empty()) { - deps = todo.front(); + resFlake = todo.front(); todo.pop(); - for (auto & nonFlake : deps.nonFlakeDeps) + for (NonFlake & nonFlake : resFlake.nonFlakeDeps) printNonFlakeInfo(nonFlake, json); - for (auto & newDeps : deps.flakeDeps) - todo.push(newDeps); + for (ResolvedFlake & newResFlake : resFlake.flakeDeps) + todo.push(newResFlake); } } }; From 3392f1b77869269580b58e4931b7a79f44799ce0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:41:06 +0200 Subject: [PATCH 0149/2548] Shut up clang warning --- src/nix/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 01b0866f2..3ec5f48d5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -57,7 +57,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs "--help-config' for a list of configuration settings.\n"; } - void printHelp(const string & programName, std::ostream & out) + void printHelp(const string & programName, std::ostream & out) override { MultiCommand::printHelp(programName, out); From cbfdea685764bf66443a999e672656c54289b8c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:41:59 +0200 Subject: [PATCH 0150/2548] fetchGit -> fetchTarball --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index f52120474..a47ca862f 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index 73e75fb29..8167f87a2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; From f8a52cc598ae9e13c6fe4f04f73e60e4e63a1975 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:53:58 +0200 Subject: [PATCH 0151/2548] nlohmann-json: 3.5.0 -> 3.6.1 https://github.com/nlohmann/json/releases/tag/v3.6.1 This fixes some clang warnings. --- src/nlohmann/json.hpp | 14648 +++++++++++++++++++++------------------- 1 file changed, 7542 insertions(+), 7106 deletions(-) diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index c9af0bed3..5003a4fa2 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,12 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 +| | |__ | | | | | | version 3.6.1 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License . SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . +Copyright (c) 2013-2019 Niels Lohmann . Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -27,12 +27,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ #define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 +#define NLOHMANN_JSON_VERSION_MINOR 6 +#define NLOHMANN_JSON_VERSION_PATCH 1 #include // all_of, find, for_each #include // assert @@ -42,79 +42,417 @@ SOFTWARE. #include // initializer_list #include // istream, ostream #include // random_access_iterator_tag +#include // unique_ptr #include // accumulate #include // string, stoi, to_string #include // declval, forward, move, pair, swap - -// #include -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string #include // vector -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ +// #include + + +#include + +// #include + + +#include // transform +#include // array +#include // and, not +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray + +// #include + + +#include // exception +#include // runtime_error +#include // to_string + +// #include + + +#include // size_t + namespace nlohmann { -/*! -@brief default JSONSerializer template argument +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer> -class basic_json; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// /*! -@brief JSON Pointer +@brief general exception of the @ref basic_json class -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. +This class is an extension of `std::exception` objects with a member @a id for +exception ids. It is used as the base class for all exceptions thrown by the +@ref basic_json class. This class can hence be used as "wildcard" to catch +exceptions. -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) +Subclasses: +- @ref parse_error for exceptions indicating a parse error +- @ref invalid_iterator for exceptions indicating errors with iterators +- @ref type_error for exceptions indicating executing a member function with + a wrong type +- @ref out_of_range for exceptions indicating access out of the defined range +- @ref other_error for exceptions indicating other library errors -@since version 2.0.0 +@internal +@note To have nothrow-copy-constructible exceptions, we internally use + `std::runtime_error` which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. +@endinternal + +@liveexample{The following code shows how arbitrary library exceptions can be +caught.,exception} + +@since version 3.0.0 */ -template -class json_pointer; +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; /*! -@brief default JSON class +@brief exception indicating a parse error -This type is the default specialization of the @ref basic_json class which -uses the standard template types. +This exception is thrown by the library when a parse error occurs. Parse errors +can occur during the deserialization of JSON text, CBOR, MessagePack, as well +as when using JSON Patch. -@since version 1.0.0 +Member @a byte holds the byte index of the last read character in the input +file. + +Exceptions have ids 1xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). + +@note For an input with n bytes, 1 is the index of the first character and n+1 + is the index of the terminating null byte or the end of file. This also + holds true when reading a byte vector (CBOR or MessagePack). + +@liveexample{The following code shows how a `parse_error` exception can be +caught.,parse_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 */ -using json = basic_json<>; +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] pos the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id_, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/*! +@brief exception indicating errors with iterators + +This exception is thrown if iterators passed to a library function do not match +the expected semantics. + +Exceptions have ids 2xx. + +name / id | example message | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@liveexample{The following code shows how an `invalid_iterator` exception can be +caught.,invalid_iterator} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id_, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id_) + what_arg; + return invalid_iterator(id_, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating executing a member function with a wrong type + +This exception is thrown in case of a type error; that is, a library function is +executed on a JSON value whose type does not match the expected semantics. + +Exceptions have ids 3xx. + +name / id | example message | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | + +@liveexample{The following code shows how a `type_error` exception can be +caught.,type_error} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref out_of_range for exceptions indicating access out of the defined range +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("type_error", id_) + what_arg; + return type_error(id_, w.c_str()); + } + + private: + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating access out of the defined range + +This exception is thrown in case a library function is called on an input +parameter that exceeds the expected range, for instance in case of array +indices or nonexisting object keys. + +Exceptions have ids 4xx. + +name / id | example message | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | + +@liveexample{The following code shows how an `out_of_range` exception can be +caught.,out_of_range} + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id_, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id_) + what_arg; + return out_of_range(id_, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating other library errors + +This exception is thrown in case of errors that cannot be classified with the +other exception types. + +Exceptions have ids 5xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@sa - @ref exception for the base class of the library exceptions +@sa - @ref parse_error for exceptions indicating a parse error +@sa - @ref invalid_iterator for exceptions indicating errors with iterators +@sa - @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa - @ref out_of_range for exceptions indicating access out of the defined range + +@liveexample{The following code shows how an `other_error` exception can be +caught.,other_error} + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("other_error", id_) + what_arg; + return other_error(id_, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; +} // namespace detail } // namespace nlohmann -#endif - // #include +#include // pair + // This file contains all internal macro definitions // You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them @@ -152,6 +490,19 @@ using json = basic_json<>; #define JSON_DEPRECATED #endif +// allow for portable nodiscard warnings +#if defined(__has_cpp_attribute) + #if __has_cpp_attribute(nodiscard) + #define JSON_NODISCARD [[nodiscard]] + #elif __has_cpp_attribute(gnu::warn_unused_result) + #define JSON_NODISCARD [[gnu::warn_unused_result]] + #else + #define JSON_NODISCARD + #endif +#else + #define JSON_NODISCARD +#endif + // allow to disable exceptions #if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception @@ -159,6 +510,7 @@ using json = basic_json<>; #define JSON_CATCH(exception) catch(exception) #define JSON_INTERNAL_CATCH(exception) catch(exception) #else + #include #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) @@ -187,8 +539,8 @@ using json = basic_json<>; // manual branch prediction #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) + #define JSON_LIKELY(x) __builtin_expect(x, 1) + #define JSON_UNLIKELY(x) __builtin_expect(x, 0) #else #define JSON_LIKELY(x) x #define JSON_UNLIKELY(x) x @@ -322,8 +674,6 @@ constexpr T static_const::value; #include // false_type, is_constructible, is_integral, is_same, true_type #include // declval -// #include - // #include @@ -389,8 +739,10 @@ struct iterator_traits::value>> using pointer = T*; using reference = T&; }; -} -} +} // namespace detail +} // namespace nlohmann + +// #include // #include @@ -412,7 +764,9 @@ struct nonesuch nonesuch() = delete; ~nonesuch() = delete; nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; }; template +// #include +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ namespace nlohmann @@ -792,385 +1210,6 @@ struct is_compatible_type } // namespace detail } // namespace nlohmann -// #include - - -#include // exception -#include // runtime_error -#include // to_string - -// #include - - -#include // size_t - -namespace nlohmann -{ -namespace detail -{ -/// struct to capture the start position of the current token -struct position_t -{ - /// the total number of characters read - std::size_t chars_read_total = 0; - /// the number of characters read in the current line - std::size_t chars_read_current_line = 0; - /// the number of lines read - std::size_t lines_read = 0; - - /// conversion to size_t to preserve SAX interface - constexpr operator size_t() const - { - return chars_read_total; - } -}; - -} -} - - -namespace nlohmann -{ -namespace detail -{ -//////////////// -// exceptions // -//////////////// - -/*! -@brief general exception of the @ref basic_json class - -This class is an extension of `std::exception` objects with a member @a id for -exception ids. It is used as the base class for all exceptions thrown by the -@ref basic_json class. This class can hence be used as "wildcard" to catch -exceptions. - -Subclasses: -- @ref parse_error for exceptions indicating a parse error -- @ref invalid_iterator for exceptions indicating errors with iterators -- @ref type_error for exceptions indicating executing a member function with - a wrong type -- @ref out_of_range for exceptions indicating access out of the defined range -- @ref other_error for exceptions indicating other library errors - -@internal -@note To have nothrow-copy-constructible exceptions, we internally use - `std::runtime_error` which can cope with arbitrary-length error messages. - Intermediate strings are built with static functions and then passed to - the actual constructor. -@endinternal - -@liveexample{The following code shows how arbitrary library exceptions can be -caught.,exception} - -@since version 3.0.0 -*/ -class exception : public std::exception -{ - public: - /// returns the explanatory string - const char* what() const noexcept override - { - return m.what(); - } - - /// the id of the exception - const int id; - - protected: - exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} - - static std::string name(const std::string& ename, int id_) - { - return "[json.exception." + ename + "." + std::to_string(id_) + "] "; - } - - private: - /// an exception object as storage for error messages - std::runtime_error m; -}; - -/*! -@brief exception indicating a parse error - -This exception is thrown by the library when a parse error occurs. Parse errors -can occur during the deserialization of JSON text, CBOR, MessagePack, as well -as when using JSON Patch. - -Member @a byte holds the byte index of the last read character in the input -file. - -Exceptions have ids 1xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. -json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. -json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. -json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. -json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. -json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. -json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. -json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. -json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. -json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. -json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. -json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. -json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). - -@note For an input with n bytes, 1 is the index of the first character and n+1 - is the index of the terminating null byte or the end of file. This also - holds true when reading a byte vector (CBOR or MessagePack). - -@liveexample{The following code shows how a `parse_error` exception can be -caught.,parse_error} - -@sa @ref exception for the base class of the library exceptions -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class parse_error : public exception -{ - public: - /*! - @brief create a parse error exception - @param[in] id_ the id of the exception - @param[in] position the position where the error occurred (or with - chars_read_total=0 if the position cannot be - determined) - @param[in] what_arg the explanatory string - @return parse_error object - */ - static parse_error create(int id_, const position_t& pos, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - position_string(pos) + ": " + what_arg; - return parse_error(id_, pos.chars_read_total, w.c_str()); - } - - static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + - ": " + what_arg; - return parse_error(id_, byte_, w.c_str()); - } - - /*! - @brief byte index of the parse error - - The byte index of the last read character in the input file. - - @note For an input with n bytes, 1 is the index of the first character and - n+1 is the index of the terminating null byte or the end of file. - This also holds true when reading a byte vector (CBOR or MessagePack). - */ - const std::size_t byte; - - private: - parse_error(int id_, std::size_t byte_, const char* what_arg) - : exception(id_, what_arg), byte(byte_) {} - - static std::string position_string(const position_t& pos) - { - return " at line " + std::to_string(pos.lines_read + 1) + - ", column " + std::to_string(pos.chars_read_current_line); - } -}; - -/*! -@brief exception indicating errors with iterators - -This exception is thrown if iterators passed to a library function do not match -the expected semantics. - -Exceptions have ids 2xx. - -name / id | example message | description ------------------------------------ | --------------- | ------------------------- -json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. -json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. -json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. -json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. -json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. -json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. -json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. -json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. -json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. -json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). - -@liveexample{The following code shows how an `invalid_iterator` exception can be -caught.,invalid_iterator} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class invalid_iterator : public exception -{ - public: - static invalid_iterator create(int id_, const std::string& what_arg) - { - std::string w = exception::name("invalid_iterator", id_) + what_arg; - return invalid_iterator(id_, w.c_str()); - } - - private: - invalid_iterator(int id_, const char* what_arg) - : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating executing a member function with a wrong type - -This exception is thrown in case of a type error; that is, a library function is -executed on a JSON value whose type does not match the expected semantics. - -Exceptions have ids 3xx. - -name / id | example message | description ------------------------------ | --------------- | ------------------------- -json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. -json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. -json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. -json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. -json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. -json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. -json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. -json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. -json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. -json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. -json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. -json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. -json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. -json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. -json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. -json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | -json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | - -@liveexample{The following code shows how a `type_error` exception can be -caught.,type_error} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref out_of_range for exceptions indicating access out of the defined range -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class type_error : public exception -{ - public: - static type_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("type_error", id_) + what_arg; - return type_error(id_, w.c_str()); - } - - private: - type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating access out of the defined range - -This exception is thrown in case a library function is called on an input -parameter that exceeds the expected range, for instance in case of array -indices or nonexisting object keys. - -Exceptions have ids 4xx. - -name / id | example message | description -------------------------------- | --------------- | ------------------------- -json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. -json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. -json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. -json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. -json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. -json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | -json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | -json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | - -@liveexample{The following code shows how an `out_of_range` exception can be -caught.,out_of_range} - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class out_of_range : public exception -{ - public: - static out_of_range create(int id_, const std::string& what_arg) - { - std::string w = exception::name("out_of_range", id_) + what_arg; - return out_of_range(id_, w.c_str()); - } - - private: - out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating other library errors - -This exception is thrown in case of errors that cannot be classified with the -other exception types. - -Exceptions have ids 5xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - -@sa @ref exception for the base class of the library exceptions -@sa @ref parse_error for exceptions indicating a parse error -@sa @ref invalid_iterator for exceptions indicating errors with iterators -@sa @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa @ref out_of_range for exceptions indicating access out of the defined range - -@liveexample{The following code shows how an `other_error` exception can be -caught.,other_error} - -@since version 3.0.0 -*/ -class other_error : public exception -{ - public: - static other_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("other_error", id_) + what_arg; - return other_error(id_, w.c_str()); - } - - private: - other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; -} // namespace detail -} // namespace nlohmann - // #include @@ -1178,6 +1217,7 @@ class other_error : public exception #include // and #include // size_t #include // uint8_t +#include // string namespace nlohmann { @@ -1249,32 +1289,6 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept } // namespace detail } // namespace nlohmann -// #include - - -#include // transform -#include // array -#include // and, not -#include // forward_list -#include // inserter, front_inserter, end -#include // map -#include // string -#include // tuple, make_tuple -#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible -#include // unordered_map -#include // pair, declval -#include // valarray - -// #include - -// #include - -// #include - -// #include - -// #include - namespace nlohmann { @@ -1629,32 +1643,28 @@ constexpr const auto& from_json = detail::static_const::va // #include +#include // copy #include // or, and, not #include // begin, end +#include // string #include // tuple, get #include // is_same, is_constructible, is_floating_point, is_enum, underlying_type #include // move, forward, declval, pair #include // valarray #include // vector -// #include - -// #include - -// #include - // #include #include // size_t -#include // string, to_string #include // input_iterator_tag +#include // string, to_string #include // tuple_size, get, tuple_element -// #include - // #include +// #include + namespace nlohmann { @@ -1700,13 +1710,13 @@ template class iteration_proxy_value } /// equality operator (needed for InputIterator) - bool operator==(const iteration_proxy_value& o) const noexcept + bool operator==(const iteration_proxy_value& o) const { return anchor == o.anchor; } /// inequality operator (needed for range-based for) - bool operator!=(const iteration_proxy_value& o) const noexcept + bool operator!=(const iteration_proxy_value& o) const { return anchor != o.anchor; } @@ -1795,6 +1805,11 @@ auto get(const nlohmann::detail::iteration_proxy_value& i) -> decl // And see https://github.com/nlohmann/json/pull/1391 namespace std { +#if defined(__clang__) + // Fix: https://github.com/nlohmann/json/issues/1401 + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wmismatched-tags" +#endif template class tuple_size<::nlohmann::detail::iteration_proxy_value> : public std::integral_constant {}; @@ -1807,7 +1822,17 @@ class tuple_element> get(std::declval < ::nlohmann::detail::iteration_proxy_value> ())); }; -} +#if defined(__clang__) + #pragma clang diagnostic pop +#endif +} // namespace std + +// #include + +// #include + +// #include + namespace nlohmann { @@ -2137,11 +2162,81 @@ constexpr const auto& to_json = detail::static_const::value; } // namespace } // namespace nlohmann + +namespace nlohmann +{ + +template +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template + static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward(j), val))) + -> decltype(::nlohmann::from_json(std::forward(j), val), void()) + { + ::nlohmann::from_json(std::forward(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template + static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward(val)))) + -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) + { + ::nlohmann::to_json(j, std::forward(val)); + } +}; + +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + + +#include // generate_n +#include // array +#include // assert +#include // ldexp +#include // size_t +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // snprintf +#include // memcpy +#include // back_inserter +#include // numeric_limits +#include // char_traits, string +#include // make_pair, move + +// #include + // #include +#include // array #include // assert #include // size_t +#include //FILE * #include // strlen #include // istream #include // begin, end, iterator_traits, random_access_iterator_tag, distance, next @@ -2150,7 +2245,8 @@ constexpr const auto& to_json = detail::static_const::value; #include // string, char_traits #include // enable_if, is_base_of, is_pointer, is_integral, remove_pointer #include // pair, declval -#include //FILE * + +// #include // #include @@ -2198,10 +2294,18 @@ class file_input_adapter : public input_adapter_protocol : m_file(f) {} + // make class move-only + file_input_adapter(const file_input_adapter&) = delete; + file_input_adapter(file_input_adapter&&) = default; + file_input_adapter& operator=(const file_input_adapter&) = delete; + file_input_adapter& operator=(file_input_adapter&&) = default; + ~file_input_adapter() override = default; + std::char_traits::int_type get_character() noexcept override { return std::fgetc(m_file); } + private: /// the file pointer to read from std::FILE* m_file; @@ -2293,7 +2397,11 @@ template struct wide_string_input_helper { // UTF-32 - static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + static void fill_buffer(const WideStringType& str, + size_t& current_wchar, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) { utf8_bytes_index = 0; @@ -2305,39 +2413,39 @@ struct wide_string_input_helper else { // get the current character - const auto wc = static_cast(str[current_wchar++]); + const auto wc = static_cast(str[current_wchar++]); // UTF-32 to UTF-8 encoding if (wc < 0x80) { - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; } else if (wc <= 0x7FF) { - utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); - utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((wc >> 6u) & 0x1Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 2; } else if (wc <= 0xFFFF) { - utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); - utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((wc >> 12u) & 0x0Fu)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 3; } else if (wc <= 0x10FFFF) { - utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); - utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); - utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast::int_type>(0xF0u | ((wc >> 18u) & 0x07u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((wc >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 4; } else { // unknown character - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; } } @@ -2348,7 +2456,11 @@ template struct wide_string_input_helper { // UTF-16 - static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + static void fill_buffer(const WideStringType& str, + size_t& current_wchar, + std::array::int_type, 4>& utf8_bytes, + size_t& utf8_bytes_index, + size_t& utf8_bytes_filled) { utf8_bytes_index = 0; @@ -2360,44 +2472,44 @@ struct wide_string_input_helper else { // get the current character - const auto wc = static_cast(str[current_wchar++]); + const auto wc = static_cast(str[current_wchar++]); // UTF-16 to UTF-8 encoding if (wc < 0x80) { - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; } else if (wc <= 0x7FF) { - utf8_bytes[0] = 0xC0 | ((wc >> 6)); - utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast::int_type>(0xC0u | ((wc >> 6u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 2; } else if (0xD800 > wc or wc >= 0xE000) { - utf8_bytes[0] = 0xE0 | ((wc >> 12)); - utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); - utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes[0] = static_cast::int_type>(0xE0u | ((wc >> 12u))); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((wc >> 6u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | (wc & 0x3Fu)); utf8_bytes_filled = 3; } else { if (current_wchar < str.size()) { - const auto wc2 = static_cast(str[current_wchar++]); - const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); - utf8_bytes[0] = 0xf0 | (charcode >> 18); - utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); - utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); - utf8_bytes[3] = 0x80 | (charcode & 0x3F); + const auto wc2 = static_cast(str[current_wchar++]); + const auto charcode = 0x10000u + (((wc & 0x3FFu) << 10u) | (wc2 & 0x3FFu)); + utf8_bytes[0] = static_cast::int_type>(0xF0u | (charcode >> 18u)); + utf8_bytes[1] = static_cast::int_type>(0x80u | ((charcode >> 12u) & 0x3Fu)); + utf8_bytes[2] = static_cast::int_type>(0x80u | ((charcode >> 6u) & 0x3Fu)); + utf8_bytes[3] = static_cast::int_type>(0x80u | (charcode & 0x3Fu)); utf8_bytes_filled = 4; } else { // unknown character ++current_wchar; - utf8_bytes[0] = wc; + utf8_bytes[0] = static_cast::int_type>(wc); utf8_bytes_filled = 1; } } @@ -2409,7 +2521,7 @@ template class wide_string_input_adapter : public input_adapter_protocol { public: - explicit wide_string_input_adapter(const WideStringType& w) noexcept + explicit wide_string_input_adapter(const WideStringType& w) noexcept : str(w) {} @@ -2561,23 +2673,2847 @@ class input_adapter } // namespace detail } // namespace nlohmann +// #include + + +#include // assert +#include +#include // string +#include // move +#include // vector + +// #include + +// #include + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + // make class move-only + json_sax_dom_parser(const json_sax_dom_parser&) = delete; + json_sax_dom_parser(json_sax_dom_parser&&) = default; + json_sax_dom_parser& operator=(const json_sax_dom_parser&) = delete; + json_sax_dom_parser& operator=(json_sax_dom_parser&&) = default; + ~json_sax_dom_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*static_cast(&ex)); + case 4: + JSON_THROW(*static_cast(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*static_cast(&ex)); + case 3: + JSON_THROW(*static_cast(&ex)); + case 5: + JSON_THROW(*static_cast(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward(v)); + return &(ref_stack.back()->m_value.array->back()); + } + + assert(ref_stack.back()->is_object()); + assert(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + // make class move-only + json_sax_dom_callback_parser(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser(json_sax_dom_callback_parser&&) = default; + json_sax_dom_callback_parser& operator=(const json_sax_dom_callback_parser&) = delete; + json_sax_dom_callback_parser& operator=(json_sax_dom_callback_parser&&) = default; + ~json_sax_dom_callback_parser() = default; + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back() and JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back() and not callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back() and ref_stack.back()->is_object()) + { + // remove discarded value + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back() and JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty() and ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*static_cast(&ex)); + case 4: + JSON_THROW(*static_cast(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*static_cast(&ex)); + case 3: + JSON_THROW(*static_cast(&ex)); + case 5: + JSON_THROW(*static_cast(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template + std::pair handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + // array + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + + // object + assert(ref_stack.back()->is_object()); + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack {}; + /// stack to manage which values to keep + std::vector keep_stack {}; + /// stack to manage which object keys to keep + std::vector key_keep_stack {}; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include + +// #include + + +#include // size_t +#include // declval +#include // string + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +using null_function_t = decltype(std::declval().null()); + +template +using boolean_function_t = + decltype(std::declval().boolean(std::declval())); + +template +using number_integer_function_t = + decltype(std::declval().number_integer(std::declval())); + +template +using number_unsigned_function_t = + decltype(std::declval().number_unsigned(std::declval())); + +template +using number_float_function_t = decltype(std::declval().number_float( + std::declval(), std::declval())); + +template +using string_function_t = + decltype(std::declval().string(std::declval())); + +template +using start_object_function_t = + decltype(std::declval().start_object(std::declval())); + +template +using key_function_t = + decltype(std::declval().key(std::declval())); + +template +using end_object_function_t = decltype(std::declval().end_object()); + +template +using start_array_function_t = + decltype(std::declval().start_array(std::declval())); + +template +using end_array_function_t = decltype(std::declval().end_array()); + +template +using parse_error_function_t = decltype(std::declval().parse_error( + std::declval(), std::declval(), + std::declval())); + +template +struct is_sax +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value && + is_detected_exact::value; +}; + +template +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts {}; + assert(ia); + } + + // make class move-only + binary_reader(const binary_reader&) = delete; + binary_reader(binary_reader&&) = default; + binary_reader& operator=(const binary_reader&) = delete; + binary_reader& operator=(binary_reader&&) = default; + ~binary_reader() = default; + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + + @return + */ + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + + // strict mode: next byte must be EOF + if (result and strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); + } + } + + return result; + } + + /*! + @brief determine system byte order + + @return true if and only if system's byte order is little endian + + @note from http://stackoverflow.com/a/1001328/266378 + */ + static constexpr bool little_endianess(int num = 1) noexcept + { + return *reinterpret_cast(&num) == 1; + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast(1), result) and get() != std::char_traits::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number(input_format_t::bson, number) and sax->number_float(static_cast(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + std::array cr{{}}; + (std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr.data()), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr.data()))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array and not sax->key(key)) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char = true) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + std::uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + std::uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + std::uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + std::uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + std::uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + std::uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + std::uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + std::uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) + - static_cast(number)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) and sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast(static_cast(current) & 0x1Fu)); + + case 0x98: // array (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(std::size_t(-1)); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast(static_cast(current) & 0x1Fu)); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(std::size_t(-1)); + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte1 << 8u) + byte2); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, static_cast(current) & 0x1Fu, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + std::uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + std::uint16_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + std::uint32_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + std::uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast(static_cast(current) & 0x0Fu)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast(static_cast(current) & 0x0Fu)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xCA: // float 32 + { + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); + } + + case 0xCB: // float 64 + { + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); + } + + case 0xCC: // uint 8 + { + std::uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + std::uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + std::uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + std::uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + std::int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD1: // int 16 + { + std::int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD2: // int 32 + { + std::int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD3: // int 64 + { + std::int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xDC: // array 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); + } + + case 0xDD: // array 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); + } + + case 0xDE: // map 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); + } + + case 0xDF: // map 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, static_cast(current) & 0x1Fu, result); + } + + case 0xD9: // str 8 + { + std::uint8_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + std::uint16_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + std::uint32_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO(niels): may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + std::uint8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + std::int8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + std::int16_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + std::int32_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + std::int64_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + std::uint8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'i': + { + std::int8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'I': + { + std::int16_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'l': + { + std::int32_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'L': + { + std::int64_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + } + + return get_ubjson_size_value(result.first); + } + + if (current == '#') + { + return get_ubjson_size_value(result.first); + } + + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + std::uint8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); + } + + case 'i': + { + std::int8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'I': + { + std::int16_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'l': + { + std::int32_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'L': + { + std::int64_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); + } + + case 'D': + { + double number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); + } + string_t s(1, static_cast(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits::eof()` in that case. + + @return character read from the input + */ + int get() + { + ++chars_read; + return current = ia->get_character(); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian != InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast(current); + } + else + { + vec[i] = static_cast(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) + { + success = false; + } + return static_cast(current); + }); + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_UNLIKELY(current == std::char_traits::eof())) + { + return sax->parse_error(chars_read, "", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + std::array cr{{}}; + (std::snprintf)(cr.data(), cr.size(), "%.2hhX", static_cast(current)); + return std::string{cr.data()}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + input_adapter_t ia = nullptr; + + /// the current character + int current = std::char_traits::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianess + const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + // #include +#include // array #include // localeconv #include // size_t -#include // strtof, strtod, strtold, strtoll, strtoull #include // snprintf +#include // strtof, strtod, strtold, strtoll, strtoull #include // initializer_list #include // char_traits, string +#include // move #include // vector -// #include - // #include // #include +// #include + namespace nlohmann { @@ -2715,22 +5651,22 @@ class lexer assert(current == 'u'); int codepoint = 0; - const auto factors = { 12, 8, 4, 0 }; + const auto factors = { 12u, 8u, 4u, 0u }; for (const auto factor : factors) { get(); if (current >= '0' and current <= '9') { - codepoint += ((current - 0x30) << factor); + codepoint += static_cast((static_cast(current) - 0x30u) << factor); } else if (current >= 'A' and current <= 'F') { - codepoint += ((current - 0x37) << factor); + codepoint += static_cast((static_cast(current) - 0x37u) << factor); } else if (current >= 'a' and current <= 'f') { - codepoint += ((current - 0x57) << factor); + codepoint += static_cast((static_cast(current) - 0x57u) << factor); } else { @@ -2888,15 +5824,15 @@ class lexer if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF)) { // overwrite codepoint - codepoint = - // high surrogate occupies the most significant 22 bits - (codepoint1 << 10) - // low surrogate occupies the least significant 15 bits - + codepoint2 - // there is still the 0xD800, 0xDC00 and 0x10000 noise - // in the result so we have to subtract with: - // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 - - 0x35FDC00; + codepoint = static_cast( + // high surrogate occupies the most significant 22 bits + (static_cast(codepoint1) << 10u) + // low surrogate occupies the least significant 15 bits + + static_cast(codepoint2) + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00u); } else { @@ -2931,23 +5867,23 @@ class lexer else if (codepoint <= 0x7FF) { // 2-byte characters: 110xxxxx 10xxxxxx - add(0xC0 | (codepoint >> 6)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast(0xC0u | (static_cast(codepoint) >> 6u))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); } else if (codepoint <= 0xFFFF) { // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx - add(0xE0 | (codepoint >> 12)); - add(0x80 | ((codepoint >> 6) & 0x3F)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast(0xE0u | (static_cast(codepoint) >> 12u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); } else { // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - add(0xF0 | (codepoint >> 18)); - add(0x80 | ((codepoint >> 12) & 0x3F)); - add(0x80 | ((codepoint >> 6) & 0x3F)); - add(0x80 | (codepoint & 0x3F)); + add(static_cast(0xF0u | (static_cast(codepoint) >> 18u))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 12u) & 0x3Fu))); + add(static_cast(0x80u | ((static_cast(codepoint) >> 6u) & 0x3Fu))); + add(static_cast(0x80u | (static_cast(codepoint) & 0x3Fu))); } break; @@ -3473,13 +6409,9 @@ class lexer goto scan_number_any1; } - // LCOV_EXCL_START - default: - { - // all other characters are rejected outside scan_number() - assert(false); - } - // LCOV_EXCL_STOP + // all other characters are rejected outside scan_number() + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } scan_number_minus: @@ -3829,7 +6761,7 @@ scan_number_done: if (current == '\n') { ++position.lines_read; - ++position.chars_read_current_line = 0; + position.chars_read_current_line = 0; } return current; @@ -3864,7 +6796,7 @@ scan_number_done: if (JSON_LIKELY(current != std::char_traits::eof())) { - assert(token_string.size() != 0); + assert(not token_string.empty()); token_string.pop_back(); } } @@ -3926,9 +6858,9 @@ scan_number_done: if ('\x00' <= c and c <= '\x1F') { // escape control characters - char cs[9]; - (std::snprintf)(cs, 9, "", static_cast(c)); - result += cs; + std::array cs{{}}; + (std::snprintf)(cs.data(), cs.size(), "", static_cast(c)); + result += cs.data(); } else { @@ -4050,7 +6982,7 @@ scan_number_done: bool next_unget = false; /// the start position of the current token - position_t position; + position_t position {}; /// raw input token string (for error messages) std::vector token_string {}; @@ -4081,865 +7013,20 @@ scan_number_done: #include // function #include // string #include // move +#include // vector // #include -// #include - -// #include - - -#include // size_t -#include // declval - -// #include - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -using null_function_t = decltype(std::declval().null()); - -template -using boolean_function_t = - decltype(std::declval().boolean(std::declval())); - -template -using number_integer_function_t = - decltype(std::declval().number_integer(std::declval())); - -template -using number_unsigned_function_t = - decltype(std::declval().number_unsigned(std::declval())); - -template -using number_float_function_t = decltype(std::declval().number_float( - std::declval(), std::declval())); - -template -using string_function_t = - decltype(std::declval().string(std::declval())); - -template -using start_object_function_t = - decltype(std::declval().start_object(std::declval())); - -template -using key_function_t = - decltype(std::declval().key(std::declval())); - -template -using end_object_function_t = decltype(std::declval().end_object()); - -template -using start_array_function_t = - decltype(std::declval().start_array(std::declval())); - -template -using end_array_function_t = decltype(std::declval().end_array()); - -template -using parse_error_function_t = decltype(std::declval().parse_error( - std::declval(), std::declval(), - std::declval())); - -template -struct is_sax -{ - private: - static_assert(is_basic_json::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using exception_t = typename BasicJsonType::exception; - - public: - static constexpr bool value = - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value && - is_detected_exact::value; -}; - -template -struct is_sax_static_asserts -{ - private: - static_assert(is_basic_json::value, - "BasicJsonType must be of type basic_json<...>"); - - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using exception_t = typename BasicJsonType::exception; - - public: - static_assert(is_detected_exact::value, - "Missing/invalid function: bool null()"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool boolean(bool)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool boolean(bool)"); - static_assert( - is_detected_exact::value, - "Missing/invalid function: bool number_integer(number_integer_t)"); - static_assert( - is_detected_exact::value, - "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); - static_assert( - is_detected_exact::value, - "Missing/invalid function: bool string(string_t&)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool start_object(std::size_t)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool key(string_t&)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool end_object()"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool start_array(std::size_t)"); - static_assert(is_detected_exact::value, - "Missing/invalid function: bool end_array()"); - static_assert( - is_detected_exact::value, - "Missing/invalid function: bool parse_error(std::size_t, const " - "std::string&, const exception&)"); -}; -} // namespace detail -} // namespace nlohmann - // #include // #include - -#include -#include -#include - -// #include - -// #include - - -namespace nlohmann -{ - -/*! -@brief SAX interface - -This class describes the SAX interface used by @ref nlohmann::json::sax_parse. -Each function is called in different situations while the input is parsed. The -boolean return value informs the parser whether to continue processing the -input. -*/ -template -struct json_sax -{ - /// type for (signed) integers - using number_integer_t = typename BasicJsonType::number_integer_t; - /// type for unsigned integers - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - /// type for floating-point numbers - using number_float_t = typename BasicJsonType::number_float_t; - /// type for strings - using string_t = typename BasicJsonType::string_t; - - /*! - @brief a null value was read - @return whether parsing should proceed - */ - virtual bool null() = 0; - - /*! - @brief a boolean value was read - @param[in] val boolean value - @return whether parsing should proceed - */ - virtual bool boolean(bool val) = 0; - - /*! - @brief an integer number was read - @param[in] val integer value - @return whether parsing should proceed - */ - virtual bool number_integer(number_integer_t val) = 0; - - /*! - @brief an unsigned integer number was read - @param[in] val unsigned integer value - @return whether parsing should proceed - */ - virtual bool number_unsigned(number_unsigned_t val) = 0; - - /*! - @brief an floating-point number was read - @param[in] val floating-point value - @param[in] s raw token value - @return whether parsing should proceed - */ - virtual bool number_float(number_float_t val, const string_t& s) = 0; - - /*! - @brief a string was read - @param[in] val string value - @return whether parsing should proceed - @note It is safe to move the passed string. - */ - virtual bool string(string_t& val) = 0; - - /*! - @brief the beginning of an object was read - @param[in] elements number of object elements or -1 if unknown - @return whether parsing should proceed - @note binary formats may report the number of elements - */ - virtual bool start_object(std::size_t elements) = 0; - - /*! - @brief an object key was read - @param[in] val object key - @return whether parsing should proceed - @note It is safe to move the passed string. - */ - virtual bool key(string_t& val) = 0; - - /*! - @brief the end of an object was read - @return whether parsing should proceed - */ - virtual bool end_object() = 0; - - /*! - @brief the beginning of an array was read - @param[in] elements number of array elements or -1 if unknown - @return whether parsing should proceed - @note binary formats may report the number of elements - */ - virtual bool start_array(std::size_t elements) = 0; - - /*! - @brief the end of an array was read - @return whether parsing should proceed - */ - virtual bool end_array() = 0; - - /*! - @brief a parse error occurred - @param[in] position the position in the input where the error occurs - @param[in] last_token the last read token - @param[in] ex an exception object describing the error - @return whether parsing should proceed (must return false) - */ - virtual bool parse_error(std::size_t position, - const std::string& last_token, - const detail::exception& ex) = 0; - - virtual ~json_sax() = default; -}; - - -namespace detail -{ -/*! -@brief SAX implementation to create a JSON value from SAX events - -This class implements the @ref json_sax interface and processes the SAX events -to create a JSON value which makes it basically a DOM parser. The structure or -hierarchy of the JSON value is managed by the stack `ref_stack` which contains -a pointer to the respective array or object for each recursion depth. - -After successful parsing, the value that is passed by reference to the -constructor contains the parsed value. - -@tparam BasicJsonType the JSON type -*/ -template -class json_sax_dom_parser -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - - /*! - @param[in, out] r reference to a JSON value that is manipulated while - parsing - @param[in] allow_exceptions_ whether parse errors yield exceptions - */ - explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) - : root(r), allow_exceptions(allow_exceptions_) - {} - - bool null() - { - handle_value(nullptr); - return true; - } - - bool boolean(bool val) - { - handle_value(val); - return true; - } - - bool number_integer(number_integer_t val) - { - handle_value(val); - return true; - } - - bool number_unsigned(number_unsigned_t val) - { - handle_value(val); - return true; - } - - bool number_float(number_float_t val, const string_t& /*unused*/) - { - handle_value(val); - return true; - } - - bool string(string_t& val) - { - handle_value(val); - return true; - } - - bool start_object(std::size_t len) - { - ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); - - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive object size: " + std::to_string(len))); - } - - return true; - } - - bool key(string_t& val) - { - // add null at given key and store the reference for later - object_element = &(ref_stack.back()->m_value.object->operator[](val)); - return true; - } - - bool end_object() - { - ref_stack.pop_back(); - return true; - } - - bool start_array(std::size_t len) - { - ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); - - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive array size: " + std::to_string(len))); - } - - return true; - } - - bool end_array() - { - ref_stack.pop_back(); - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, - const detail::exception& ex) - { - errored = true; - if (allow_exceptions) - { - // determine the proper exception type from the id - switch ((ex.id / 100) % 100) - { - case 1: - JSON_THROW(*reinterpret_cast(&ex)); - case 4: - JSON_THROW(*reinterpret_cast(&ex)); - // LCOV_EXCL_START - case 2: - JSON_THROW(*reinterpret_cast(&ex)); - case 3: - JSON_THROW(*reinterpret_cast(&ex)); - case 5: - JSON_THROW(*reinterpret_cast(&ex)); - default: - assert(false); - // LCOV_EXCL_STOP - } - } - return false; - } - - constexpr bool is_errored() const - { - return errored; - } - - private: - /*! - @invariant If the ref stack is empty, then the passed value will be the new - root. - @invariant If the ref stack contains a value, then it is an array or an - object to which we can add elements - */ - template - BasicJsonType* handle_value(Value&& v) - { - if (ref_stack.empty()) - { - root = BasicJsonType(std::forward(v)); - return &root; - } - - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->emplace_back(std::forward(v)); - return &(ref_stack.back()->m_value.array->back()); - } - else - { - assert(object_element); - *object_element = BasicJsonType(std::forward(v)); - return object_element; - } - } - - /// the parsed JSON value - BasicJsonType& root; - /// stack to model hierarchy of values - std::vector ref_stack; - /// helper to hold the reference for the next object element - BasicJsonType* object_element = nullptr; - /// whether a syntax error occurred - bool errored = false; - /// whether to throw exceptions in case of errors - const bool allow_exceptions = true; -}; - -template -class json_sax_dom_callback_parser -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using parser_callback_t = typename BasicJsonType::parser_callback_t; - using parse_event_t = typename BasicJsonType::parse_event_t; - - json_sax_dom_callback_parser(BasicJsonType& r, - const parser_callback_t cb, - const bool allow_exceptions_ = true) - : root(r), callback(cb), allow_exceptions(allow_exceptions_) - { - keep_stack.push_back(true); - } - - bool null() - { - handle_value(nullptr); - return true; - } - - bool boolean(bool val) - { - handle_value(val); - return true; - } - - bool number_integer(number_integer_t val) - { - handle_value(val); - return true; - } - - bool number_unsigned(number_unsigned_t val) - { - handle_value(val); - return true; - } - - bool number_float(number_float_t val, const string_t& /*unused*/) - { - handle_value(val); - return true; - } - - bool string(string_t& val) - { - handle_value(val); - return true; - } - - bool start_object(std::size_t len) - { - // check callback for object start - const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::object_start, discarded); - keep_stack.push_back(keep); - - auto val = handle_value(BasicJsonType::value_t::object, true); - ref_stack.push_back(val.second); - - // check object limit - if (ref_stack.back()) - { - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive object size: " + std::to_string(len))); - } - } - - return true; - } - - bool key(string_t& val) - { - BasicJsonType k = BasicJsonType(val); - - // check callback for key - const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::key, k); - key_keep_stack.push_back(keep); - - // add discarded value at given key and store the reference for later - if (keep and ref_stack.back()) - { - object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); - } - - return true; - } - - bool end_object() - { - if (ref_stack.back()) - { - if (not callback(static_cast(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) - { - // discard object - *ref_stack.back() = discarded; - } - } - - assert(not ref_stack.empty()); - assert(not keep_stack.empty()); - ref_stack.pop_back(); - keep_stack.pop_back(); - - if (not ref_stack.empty() and ref_stack.back()) - { - // remove discarded value - if (ref_stack.back()->is_object()) - { - for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) - { - if (it->is_discarded()) - { - ref_stack.back()->erase(it); - break; - } - } - } - } - - return true; - } - - bool start_array(std::size_t len) - { - const bool keep = callback(static_cast(ref_stack.size()), parse_event_t::array_start, discarded); - keep_stack.push_back(keep); - - auto val = handle_value(BasicJsonType::value_t::array, true); - ref_stack.push_back(val.second); - - // check array limit - if (ref_stack.back()) - { - if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) - { - JSON_THROW(out_of_range::create(408, - "excessive array size: " + std::to_string(len))); - } - } - - return true; - } - - bool end_array() - { - bool keep = true; - - if (ref_stack.back()) - { - keep = callback(static_cast(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); - if (not keep) - { - // discard array - *ref_stack.back() = discarded; - } - } - - assert(not ref_stack.empty()); - assert(not keep_stack.empty()); - ref_stack.pop_back(); - keep_stack.pop_back(); - - // remove discarded value - if (not keep and not ref_stack.empty()) - { - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->pop_back(); - } - } - - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, - const detail::exception& ex) - { - errored = true; - if (allow_exceptions) - { - // determine the proper exception type from the id - switch ((ex.id / 100) % 100) - { - case 1: - JSON_THROW(*reinterpret_cast(&ex)); - case 4: - JSON_THROW(*reinterpret_cast(&ex)); - // LCOV_EXCL_START - case 2: - JSON_THROW(*reinterpret_cast(&ex)); - case 3: - JSON_THROW(*reinterpret_cast(&ex)); - case 5: - JSON_THROW(*reinterpret_cast(&ex)); - default: - assert(false); - // LCOV_EXCL_STOP - } - } - return false; - } - - constexpr bool is_errored() const - { - return errored; - } - - private: - /*! - @param[in] v value to add to the JSON value we build during parsing - @param[in] skip_callback whether we should skip calling the callback - function; this is required after start_array() and - start_object() SAX events, because otherwise we would call the - callback function with an empty array or object, respectively. - - @invariant If the ref stack is empty, then the passed value will be the new - root. - @invariant If the ref stack contains a value, then it is an array or an - object to which we can add elements - - @return pair of boolean (whether value should be kept) and pointer (to the - passed value in the ref_stack hierarchy; nullptr if not kept) - */ - template - std::pair handle_value(Value&& v, const bool skip_callback = false) - { - assert(not keep_stack.empty()); - - // do not handle this value if we know it would be added to a discarded - // container - if (not keep_stack.back()) - { - return {false, nullptr}; - } - - // create value - auto value = BasicJsonType(std::forward(v)); - - // check callback - const bool keep = skip_callback or callback(static_cast(ref_stack.size()), parse_event_t::value, value); - - // do not handle this value if we just learnt it shall be discarded - if (not keep) - { - return {false, nullptr}; - } - - if (ref_stack.empty()) - { - root = std::move(value); - return {true, &root}; - } - - // skip this value if we already decided to skip the parent - // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) - if (not ref_stack.back()) - { - return {false, nullptr}; - } - - // we now only expect arrays and objects - assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); - - if (ref_stack.back()->is_array()) - { - ref_stack.back()->m_value.array->push_back(std::move(value)); - return {true, &(ref_stack.back()->m_value.array->back())}; - } - else - { - // check if we should store an element for the current key - assert(not key_keep_stack.empty()); - const bool store_element = key_keep_stack.back(); - key_keep_stack.pop_back(); - - if (not store_element) - { - return {false, nullptr}; - } - - assert(object_element); - *object_element = std::move(value); - return {true, object_element}; - } - } - - /// the parsed JSON value - BasicJsonType& root; - /// stack to model hierarchy of values - std::vector ref_stack; - /// stack to manage which values to keep - std::vector keep_stack; - /// stack to manage which object keys to keep - std::vector key_keep_stack; - /// helper to hold the reference for the next object element - BasicJsonType* object_element = nullptr; - /// whether a syntax error occurred - bool errored = false; - /// callback function - const parser_callback_t callback = nullptr; - /// whether to throw exceptions in case of errors - const bool allow_exceptions = true; - /// a discarded value for the callback - BasicJsonType discarded = BasicJsonType::value_t::discarded; -}; - -template -class json_sax_acceptor -{ - public: - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - - bool null() - { - return true; - } - - bool boolean(bool /*unused*/) - { - return true; - } - - bool number_integer(number_integer_t /*unused*/) - { - return true; - } - - bool number_unsigned(number_unsigned_t /*unused*/) - { - return true; - } - - bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) - { - return true; - } - - bool string(string_t& /*unused*/) - { - return true; - } - - bool start_object(std::size_t /*unused*/ = std::size_t(-1)) - { - return true; - } - - bool key(string_t& /*unused*/) - { - return true; - } - - bool end_object() - { - return true; - } - - bool start_array(std::size_t /*unused*/ = std::size_t(-1)) - { - return true; - } - - bool end_array() - { - return true; - } - - bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) - { - return false; - } -}; -} // namespace detail - -} // namespace nlohmann - // #include +// #include + +// #include + // #include @@ -5189,14 +7276,13 @@ class parser m_lexer.get_token_string(), out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); } - else + + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) { - if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) - { - return false; - } - break; + return false; } + + break; } case token_type::literal_false: @@ -5282,103 +7368,95 @@ class parser // empty stack: we reached the end of the hierarchy: done return true; } - else + + if (states.back()) // array { - if (states.back()) // array + // comma -> next value + if (get_token() == token_type::value_separator) { - // comma -> next value - if (get_token() == token_type::value_separator) + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) + { + if (JSON_UNLIKELY(not sax->end_array())) { - // parse a new value - get_token(); - continue; + return false; } - // closing ] - if (JSON_LIKELY(last_token == token_type::end_array)) - { - if (JSON_UNLIKELY(not sax->end_array())) - { - return false; - } + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } - // We are done with this array. Before we can parse a - // new value, we need to evaluate the new state first. - // By setting skip_to_state_evaluation to false, we - // are effectively jumping to the beginning of this if. - assert(not states.empty()); - states.pop_back(); - skip_to_state_evaluation = true; - continue; - } - else + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); + } + else // object + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_array, "array"))); + exception_message(token_type::value_string, "object key"))); } - } - else // object - { - // comma -> next value - if (get_token() == token_type::value_separator) + + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) { - // parse key - if (JSON_UNLIKELY(get_token() != token_type::value_string)) - { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::value_string, "object key"))); - } - else - { - if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) - { - return false; - } - } - - // parse separator (:) - if (JSON_UNLIKELY(get_token() != token_type::name_separator)) - { - return sax->parse_error(m_lexer.get_position(), - m_lexer.get_token_string(), - parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::name_separator, "object separator"))); - } - - // parse values - get_token(); - continue; + return false; } - // closing } - if (JSON_LIKELY(last_token == token_type::end_object)) - { - if (JSON_UNLIKELY(not sax->end_object())) - { - return false; - } - - // We are done with this object. Before we can parse a - // new value, we need to evaluate the new state first. - // By setting skip_to_state_evaluation to false, we - // are effectively jumping to the beginning of this if. - assert(not states.empty()); - states.pop_back(); - skip_to_state_evaluation = true; - continue; - } - else + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) { return sax->parse_error(m_lexer.get_position(), m_lexer.get_token_string(), parse_error::create(101, m_lexer.get_position(), - exception_message(token_type::end_object, "object"))); + exception_message(token_type::name_separator, "object separator"))); } + + // parse values + get_token(); + continue; } + + // closing } + if (JSON_LIKELY(last_token == token_type::end_object)) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); } } } @@ -5386,7 +7464,7 @@ class parser /// get next token from lexer token_type get_token() { - return (last_token = m_lexer.scan()); + return last_token = m_lexer.scan(); } std::string exception_message(const token_type expected, const std::string& context) @@ -5431,6 +7509,9 @@ class parser } // namespace detail } // namespace nlohmann +// #include + + // #include @@ -5553,11 +7634,6 @@ class primitive_iterator_t } // namespace detail } // namespace nlohmann -// #include - - -// #include - namespace nlohmann { @@ -5598,6 +7674,8 @@ template struct internal_iterator // #include +// #include + // #include @@ -6196,10 +8274,11 @@ class iter_impl /// associated JSON instance pointer m_object = nullptr; /// the actual iterator of the associated instance - internal_iterator::type> m_it; + internal_iterator::type> m_it {}; }; } // namespace detail } // namespace nlohmann + // #include // #include @@ -6323,5390 +8402,22 @@ class json_reverse_iterator : public std::reverse_iterator } // namespace detail } // namespace nlohmann -// #include - - -#include // copy -#include // size_t -#include // streamsize -#include // back_inserter -#include // shared_ptr, make_shared -#include // basic_ostream -#include // basic_string -#include // vector - -namespace nlohmann -{ -namespace detail -{ -/// abstract output adapter interface -template struct output_adapter_protocol -{ - virtual void write_character(CharType c) = 0; - virtual void write_characters(const CharType* s, std::size_t length) = 0; - virtual ~output_adapter_protocol() = default; -}; - -/// a type to simplify interfaces -template -using output_adapter_t = std::shared_ptr>; - -/// output adapter for byte vectors -template -class output_vector_adapter : public output_adapter_protocol -{ - public: - explicit output_vector_adapter(std::vector& vec) noexcept - : v(vec) - {} - - void write_character(CharType c) override - { - v.push_back(c); - } - - void write_characters(const CharType* s, std::size_t length) override - { - std::copy(s, s + length, std::back_inserter(v)); - } - - private: - std::vector& v; -}; - -/// output adapter for output streams -template -class output_stream_adapter : public output_adapter_protocol -{ - public: - explicit output_stream_adapter(std::basic_ostream& s) noexcept - : stream(s) - {} - - void write_character(CharType c) override - { - stream.put(c); - } - - void write_characters(const CharType* s, std::size_t length) override - { - stream.write(s, static_cast(length)); - } - - private: - std::basic_ostream& stream; -}; - -/// output adapter for basic_string -template> -class output_string_adapter : public output_adapter_protocol -{ - public: - explicit output_string_adapter(StringType& s) noexcept - : str(s) - {} - - void write_character(CharType c) override - { - str.push_back(c); - } - - void write_characters(const CharType* s, std::size_t length) override - { - str.append(s, length); - } - - private: - StringType& str; -}; - -template> -class output_adapter -{ - public: - output_adapter(std::vector& vec) - : oa(std::make_shared>(vec)) {} - - output_adapter(std::basic_ostream& s) - : oa(std::make_shared>(s)) {} - - output_adapter(StringType& s) - : oa(std::make_shared>(s)) {} - - operator output_adapter_t() - { - return oa; - } - - private: - output_adapter_t oa = nullptr; -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // generate_n -#include // array -#include // assert -#include // ldexp -#include // size_t -#include // uint8_t, uint16_t, uint32_t, uint64_t -#include // snprintf -#include // memcpy -#include // back_inserter -#include // numeric_limits -#include // char_traits, string -#include // make_pair, move - -// #include - -// #include - -// #include - -// #include - -// #include - -// #include - - -namespace nlohmann -{ -namespace detail -{ -/////////////////// -// binary reader // -/////////////////// - -/*! -@brief deserialization of CBOR, MessagePack, and UBJSON values -*/ -template> -class binary_reader -{ - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - using number_float_t = typename BasicJsonType::number_float_t; - using string_t = typename BasicJsonType::string_t; - using json_sax_t = SAX; - - public: - /*! - @brief create a binary reader - - @param[in] adapter input adapter to read from - */ - explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) - { - (void)detail::is_sax_static_asserts {}; - assert(ia); - } - - /*! - @param[in] format the binary format to parse - @param[in] sax_ a SAX event processor - @param[in] strict whether to expect the input to be consumed completed - - @return - */ - bool sax_parse(const input_format_t format, - json_sax_t* sax_, - const bool strict = true) - { - sax = sax_; - bool result = false; - - switch (format) - { - case input_format_t::bson: - result = parse_bson_internal(); - break; - - case input_format_t::cbor: - result = parse_cbor_internal(); - break; - - case input_format_t::msgpack: - result = parse_msgpack_internal(); - break; - - case input_format_t::ubjson: - result = parse_ubjson_internal(); - break; - - // LCOV_EXCL_START - default: - assert(false); - // LCOV_EXCL_STOP - } - - // strict mode: next byte must be EOF - if (result and strict) - { - if (format == input_format_t::ubjson) - { - get_ignore_noop(); - } - else - { - get(); - } - - if (JSON_UNLIKELY(current != std::char_traits::eof())) - { - return sax->parse_error(chars_read, get_token_string(), - parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); - } - } - - return result; - } - - /*! - @brief determine system byte order - - @return true if and only if system's byte order is little endian - - @note from http://stackoverflow.com/a/1001328/266378 - */ - static constexpr bool little_endianess(int num = 1) noexcept - { - return (*reinterpret_cast(&num) == 1); - } - - private: - ////////// - // BSON // - ////////// - - /*! - @brief Reads in a BSON-object and passes it to the SAX-parser. - @return whether a valid BSON-value was passed to the SAX parser - */ - bool parse_bson_internal() - { - std::int32_t document_size; - get_number(input_format_t::bson, document_size); - - if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) - { - return false; - } - - return sax->end_object(); - } - - /*! - @brief Parses a C-style string from the BSON input. - @param[in, out] result A reference to the string variable where the read - string is to be stored. - @return `true` if the \x00-byte indicating the end of the string was - encountered before the EOF; false` indicates an unexpected EOF. - */ - bool get_bson_cstr(string_t& result) - { - auto out = std::back_inserter(result); - while (true) - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) - { - return false; - } - if (current == 0x00) - { - return true; - } - *out++ = static_cast(current); - } - - return true; - } - - /*! - @brief Parses a zero-terminated string of length @a len from the BSON - input. - @param[in] len The length (including the zero-byte at the end) of the - string to be read. - @param[in, out] result A reference to the string variable where the read - string is to be stored. - @tparam NumberType The type of the length @a len - @pre len >= 1 - @return `true` if the string was successfully parsed - */ - template - bool get_bson_string(const NumberType len, string_t& result) - { - if (JSON_UNLIKELY(len < 1)) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); - } - - return get_string(input_format_t::bson, len - static_cast(1), result) and get() != std::char_traits::eof(); - } - - /*! - @brief Read a BSON document element of the given @a element_type. - @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html - @param[in] element_type_parse_position The position in the input stream, - where the `element_type` was read. - @warning Not all BSON element types are supported yet. An unsupported - @a element_type will give rise to a parse_error.114: - Unsupported BSON record type 0x... - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_internal(const int element_type, - const std::size_t element_type_parse_position) - { - switch (element_type) - { - case 0x01: // double - { - double number; - return get_number(input_format_t::bson, number) and sax->number_float(static_cast(number), ""); - } - - case 0x02: // string - { - std::int32_t len; - string_t value; - return get_number(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); - } - - case 0x03: // object - { - return parse_bson_internal(); - } - - case 0x04: // array - { - return parse_bson_array(); - } - - case 0x08: // boolean - { - return sax->boolean(get() != 0); - } - - case 0x0A: // null - { - return sax->null(); - } - - case 0x10: // int32 - { - std::int32_t value; - return get_number(input_format_t::bson, value) and sax->number_integer(value); - } - - case 0x12: // int64 - { - std::int64_t value; - return get_number(input_format_t::bson, value) and sax->number_integer(value); - } - - default: // anything else not supported (yet) - { - char cr[3]; - (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast(element_type)); - return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); - } - } - } - - /*! - @brief Read a BSON element list (as specified in the BSON-spec) - - The same binary layout is used for objects and arrays, hence it must be - indicated with the argument @a is_array which one is expected - (true --> array, false --> object). - - @param[in] is_array Determines if the element list being read is to be - treated as an object (@a is_array == false), or as an - array (@a is_array == true). - @return whether a valid BSON-object/array was passed to the SAX parser - */ - bool parse_bson_element_list(const bool is_array) - { - string_t key; - while (int element_type = get()) - { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) - { - return false; - } - - const std::size_t element_type_parse_position = chars_read; - if (JSON_UNLIKELY(not get_bson_cstr(key))) - { - return false; - } - - if (not is_array) - { - if (not sax->key(key)) - { - return false; - } - } - - if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) - { - return false; - } - - // get_bson_cstr only appends - key.clear(); - } - - return true; - } - - /*! - @brief Reads an array from the BSON input and passes it to the SAX-parser. - @return whether a valid BSON-array was passed to the SAX parser - */ - bool parse_bson_array() - { - std::int32_t document_size; - get_number(input_format_t::bson, document_size); - - if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) - { - return false; - } - - return sax->end_array(); - } - - ////////// - // CBOR // - ////////// - - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether a valid CBOR value was passed to the SAX parser - */ - bool parse_cbor_internal(const bool get_char = true) - { - switch (get_char ? get() : current) - { - // EOF - case std::char_traits::eof(): - return unexpect_eof(input_format_t::cbor, "value"); - - // Integer 0x00..0x17 (0..23) - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - return sax->number_unsigned(static_cast(current)); - - case 0x18: // Unsigned integer (one-byte uint8_t follows) - { - uint8_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } - - case 0x19: // Unsigned integer (two-byte uint16_t follows) - { - uint16_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } - - case 0x1A: // Unsigned integer (four-byte uint32_t follows) - { - uint32_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } - - case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - { - uint64_t number; - return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); - } - - // Negative integer -1-0x00..-1-0x17 (-1..-24) - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - return sax->number_integer(static_cast(0x20 - 1 - current)); - - case 0x38: // Negative integer (one-byte uint8_t follows) - { - uint8_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); - } - - case 0x39: // Negative integer -1-n (two-byte uint16_t follows) - { - uint16_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); - } - - case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) - { - uint32_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - number); - } - - case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) - { - uint64_t number; - return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast(-1) - - static_cast(number)); - } - - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) - case 0x7F: // UTF-8 string (indefinite length) - { - string_t s; - return get_cbor_string(s) and sax->string(s); - } - - // array (0x00..0x17 data items follow) - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - return get_cbor_array(static_cast(current & 0x1F)); - - case 0x98: // array (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); - } - - case 0x99: // array (two-byte uint16_t for n follow) - { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); - } - - case 0x9A: // array (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); - } - - case 0x9B: // array (eight-byte uint64_t for n follow) - { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast(len)); - } - - case 0x9F: // array (indefinite length) - return get_cbor_array(std::size_t(-1)); - - // map (0x00..0x17 pairs of data items follow) - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - return get_cbor_object(static_cast(current & 0x1F)); - - case 0xB8: // map (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); - } - - case 0xB9: // map (two-byte uint16_t for n follow) - { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); - } - - case 0xBA: // map (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); - } - - case 0xBB: // map (eight-byte uint64_t for n follow) - { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast(len)); - } - - case 0xBF: // map (indefinite length) - return get_cbor_object(std::size_t(-1)); - - case 0xF4: // false - return sax->boolean(false); - - case 0xF5: // true - return sax->boolean(true); - - case 0xF6: // null - return sax->null(); - - case 0xF9: // Half-Precision Float (two-byte IEEE 754) - { - const int byte1_raw = get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) - { - return false; - } - const int byte2_raw = get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) - { - return false; - } - - const auto byte1 = static_cast(byte1_raw); - const auto byte2 = static_cast(byte2_raw); - - // code from RFC 7049, Appendix D, Figure 3: - // As half-precision floating-point numbers were only added - // to IEEE 754 in 2008, today's programming platforms often - // still only have limited support for them. It is very - // easy to include at least decoding support for them even - // without such support. An example of a small decoder for - // half-precision floating-point numbers in the C language - // is shown in Fig. 3. - const int half = (byte1 << 8) + byte2; - const double val = [&half] - { - const int exp = (half >> 10) & 0x1F; - const int mant = half & 0x3FF; - assert(0 <= exp and exp <= 32); - assert(0 <= mant and mant <= 1024); - switch (exp) - { - case 0: - return std::ldexp(mant, -24); - case 31: - return (mant == 0) - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - default: - return std::ldexp(mant + 1024, exp - 25); - } - }(); - return sax->number_float((half & 0x8000) != 0 - ? static_cast(-val) - : static_cast(val), ""); - } - - case 0xFA: // Single-Precision Float (four-byte IEEE 754) - { - float number; - return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); - } - - case 0xFB: // Double-Precision Float (eight-byte IEEE 754) - { - double number; - return get_number(input_format_t::cbor, number) and sax->number_float(static_cast(number), ""); - } - - default: // anything else (0xFF is handled inside the other types) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); - } - } - } - - /*! - @brief reads a CBOR string - - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. - - @param[out] result created string - - @return whether string creation completed - */ - bool get_cbor_string(string_t& result) - { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) - { - return false; - } - - switch (current) - { - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - { - return get_string(input_format_t::cbor, current & 0x1F, result); - } - - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) - { - uint8_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } - - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) - { - uint16_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } - - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) - { - uint32_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } - - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) - { - uint64_t len; - return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); - } - - case 0x7F: // UTF-8 string (indefinite length) - { - while (get() != 0xFF) - { - string_t chunk; - if (not get_cbor_string(chunk)) - { - return false; - } - result.append(chunk); - } - return true; - } - - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); - } - } - } - - /*! - @param[in] len the length of the array or std::size_t(-1) for an - array of indefinite size - @return whether array creation completed - */ - bool get_cbor_array(const std::size_t len) - { - if (JSON_UNLIKELY(not sax->start_array(len))) - { - return false; - } - - if (len != std::size_t(-1)) - { - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - } - } - else - { - while (get() != 0xFF) - { - if (JSON_UNLIKELY(not parse_cbor_internal(false))) - { - return false; - } - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object or std::size_t(-1) for an - object of indefinite size - @return whether object creation completed - */ - bool get_cbor_object(const std::size_t len) - { - if (not JSON_UNLIKELY(sax->start_object(len))) - { - return false; - } - - string_t key; - if (len != std::size_t(-1)) - { - for (std::size_t i = 0; i < len; ++i) - { - get(); - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - key.clear(); - } - } - else - { - while (get() != 0xFF) - { - if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_cbor_internal())) - { - return false; - } - key.clear(); - } - } - - return sax->end_object(); - } - - ///////////// - // MsgPack // - ///////////// - - /*! - @return whether a valid MessagePack value was passed to the SAX parser - */ - bool parse_msgpack_internal() - { - switch (get()) - { - // EOF - case std::char_traits::eof(): - return unexpect_eof(input_format_t::msgpack, "value"); - - // positive fixint - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x08: - case 0x09: - case 0x0A: - case 0x0B: - case 0x0C: - case 0x0D: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - case 0x20: - case 0x21: - case 0x22: - case 0x23: - case 0x24: - case 0x25: - case 0x26: - case 0x27: - case 0x28: - case 0x29: - case 0x2A: - case 0x2B: - case 0x2C: - case 0x2D: - case 0x2E: - case 0x2F: - case 0x30: - case 0x31: - case 0x32: - case 0x33: - case 0x34: - case 0x35: - case 0x36: - case 0x37: - case 0x38: - case 0x39: - case 0x3A: - case 0x3B: - case 0x3C: - case 0x3D: - case 0x3E: - case 0x3F: - case 0x40: - case 0x41: - case 0x42: - case 0x43: - case 0x44: - case 0x45: - case 0x46: - case 0x47: - case 0x48: - case 0x49: - case 0x4A: - case 0x4B: - case 0x4C: - case 0x4D: - case 0x4E: - case 0x4F: - case 0x50: - case 0x51: - case 0x52: - case 0x53: - case 0x54: - case 0x55: - case 0x56: - case 0x57: - case 0x58: - case 0x59: - case 0x5A: - case 0x5B: - case 0x5C: - case 0x5D: - case 0x5E: - case 0x5F: - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: - case 0x78: - case 0x79: - case 0x7A: - case 0x7B: - case 0x7C: - case 0x7D: - case 0x7E: - case 0x7F: - return sax->number_unsigned(static_cast(current)); - - // fixmap - case 0x80: - case 0x81: - case 0x82: - case 0x83: - case 0x84: - case 0x85: - case 0x86: - case 0x87: - case 0x88: - case 0x89: - case 0x8A: - case 0x8B: - case 0x8C: - case 0x8D: - case 0x8E: - case 0x8F: - return get_msgpack_object(static_cast(current & 0x0F)); - - // fixarray - case 0x90: - case 0x91: - case 0x92: - case 0x93: - case 0x94: - case 0x95: - case 0x96: - case 0x97: - case 0x98: - case 0x99: - case 0x9A: - case 0x9B: - case 0x9C: - case 0x9D: - case 0x9E: - case 0x9F: - return get_msgpack_array(static_cast(current & 0x0F)); - - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - { - string_t s; - return get_msgpack_string(s) and sax->string(s); - } - - case 0xC0: // nil - return sax->null(); - - case 0xC2: // false - return sax->boolean(false); - - case 0xC3: // true - return sax->boolean(true); - - case 0xCA: // float 32 - { - float number; - return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); - } - - case 0xCB: // float 64 - { - double number; - return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast(number), ""); - } - - case 0xCC: // uint 8 - { - uint8_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCD: // uint 16 - { - uint16_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCE: // uint 32 - { - uint32_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xCF: // uint 64 - { - uint64_t number; - return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); - } - - case 0xD0: // int 8 - { - int8_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD1: // int 16 - { - int16_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD2: // int 32 - { - int32_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD3: // int 64 - { - int64_t number; - return get_number(input_format_t::msgpack, number) and sax->number_integer(number); - } - - case 0xD9: // str 8 - case 0xDA: // str 16 - case 0xDB: // str 32 - { - string_t s; - return get_msgpack_string(s) and sax->string(s); - } - - case 0xDC: // array 16 - { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); - } - - case 0xDD: // array 32 - { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast(len)); - } - - case 0xDE: // map 16 - { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); - } - - case 0xDF: // map 32 - { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast(len)); - } - - // negative fixint - case 0xE0: - case 0xE1: - case 0xE2: - case 0xE3: - case 0xE4: - case 0xE5: - case 0xE6: - case 0xE7: - case 0xE8: - case 0xE9: - case 0xEA: - case 0xEB: - case 0xEC: - case 0xED: - case 0xEE: - case 0xEF: - case 0xF0: - case 0xF1: - case 0xF2: - case 0xF3: - case 0xF4: - case 0xF5: - case 0xF6: - case 0xF7: - case 0xF8: - case 0xF9: - case 0xFA: - case 0xFB: - case 0xFC: - case 0xFD: - case 0xFE: - case 0xFF: - return sax->number_integer(static_cast(current)); - - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); - } - } - } - - /*! - @brief reads a MessagePack string - - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - - @param[out] result created string - - @return whether string creation completed - */ - bool get_msgpack_string(string_t& result) - { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) - { - return false; - } - - switch (current) - { - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: - { - return get_string(input_format_t::msgpack, current & 0x1F, result); - } - - case 0xD9: // str 8 - { - uint8_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); - } - - case 0xDA: // str 16 - { - uint16_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); - } - - case 0xDB: // str 32 - { - uint32_t len; - return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); - } - - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); - } - } - } - - /*! - @param[in] len the length of the array - @return whether array creation completed - */ - bool get_msgpack_array(const std::size_t len) - { - if (JSON_UNLIKELY(not sax->start_array(len))) - { - return false; - } - - for (std::size_t i = 0; i < len; ++i) - { - if (JSON_UNLIKELY(not parse_msgpack_internal())) - { - return false; - } - } - - return sax->end_array(); - } - - /*! - @param[in] len the length of the object - @return whether object creation completed - */ - bool get_msgpack_object(const std::size_t len) - { - if (JSON_UNLIKELY(not sax->start_object(len))) - { - return false; - } - - string_t key; - for (std::size_t i = 0; i < len; ++i) - { - get(); - if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) - { - return false; - } - - if (JSON_UNLIKELY(not parse_msgpack_internal())) - { - return false; - } - key.clear(); - } - - return sax->end_object(); - } - - //////////// - // UBJSON // - //////////// - - /*! - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether a valid UBJSON value was passed to the SAX parser - */ - bool parse_ubjson_internal(const bool get_char = true) - { - return get_ubjson_value(get_char ? get_ignore_noop() : current); - } - - /*! - @brief reads a UBJSON string - - This function is either called after reading the 'S' byte explicitly - indicating a string, or in case of an object key where the 'S' byte can be - left out. - - @param[out] result created string - @param[in] get_char whether a new character should be retrieved from the - input (true, default) or whether the last read - character should be considered instead - - @return whether string creation completed - */ - bool get_ubjson_string(string_t& result, const bool get_char = true) - { - if (get_char) - { - get(); // TODO: may we ignore N here? - } - - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) - { - return false; - } - - switch (current) - { - case 'U': - { - uint8_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'i': - { - int8_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'I': - { - int16_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'l': - { - int32_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - case 'L': - { - int64_t len; - return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); - } - - default: - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); - } - } - - /*! - @param[out] result determined size - @return whether size determination completed - */ - bool get_ubjson_size_value(std::size_t& result) - { - switch (get_ignore_noop()) - { - case 'U': - { - uint8_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast(number); - return true; - } - - case 'i': - { - int8_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast(number); - return true; - } - - case 'I': - { - int16_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast(number); - return true; - } - - case 'l': - { - int32_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast(number); - return true; - } - - case 'L': - { - int64_t number; - if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) - { - return false; - } - result = static_cast(number); - return true; - } - - default: - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); - } - } - } - - /*! - @brief determine the type and size for a container - - In the optimized UBJSON format, a type and a size can be provided to allow - for a more compact representation. - - @param[out] result pair of the size and the type - - @return whether pair creation completed - */ - bool get_ubjson_size_type(std::pair& result) - { - result.first = string_t::npos; // size - result.second = 0; // type - - get_ignore_noop(); - - if (current == '$') - { - result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) - { - return false; - } - - get_ignore_noop(); - if (JSON_UNLIKELY(current != '#')) - { - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) - { - return false; - } - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); - } - - return get_ubjson_size_value(result.first); - } - else if (current == '#') - { - return get_ubjson_size_value(result.first); - } - return true; - } - - /*! - @param prefix the previously read or set type prefix - @return whether value creation completed - */ - bool get_ubjson_value(const int prefix) - { - switch (prefix) - { - case std::char_traits::eof(): // EOF - return unexpect_eof(input_format_t::ubjson, "value"); - - case 'T': // true - return sax->boolean(true); - case 'F': // false - return sax->boolean(false); - - case 'Z': // null - return sax->null(); - - case 'U': - { - uint8_t number; - return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); - } - - case 'i': - { - int8_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } - - case 'I': - { - int16_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } - - case 'l': - { - int32_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } - - case 'L': - { - int64_t number; - return get_number(input_format_t::ubjson, number) and sax->number_integer(number); - } - - case 'd': - { - float number; - return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); - } - - case 'D': - { - double number; - return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast(number), ""); - } - - case 'C': // char - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) - { - return false; - } - if (JSON_UNLIKELY(current > 127)) - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); - } - string_t s(1, static_cast(current)); - return sax->string(s); - } - - case 'S': // string - { - string_t s; - return get_ubjson_string(s) and sax->string(s); - } - - case '[': // array - return get_ubjson_array(); - - case '{': // object - return get_ubjson_object(); - - default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); - } - } - } - - /*! - @return whether array creation completed - */ - bool get_ubjson_array() - { - std::pair size_and_type; - if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) - { - return false; - } - - if (size_and_type.first != string_t::npos) - { - if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - if (size_and_type.second != 'N') - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) - { - return false; - } - } - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - } - } - } - else - { - if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) - { - return false; - } - - while (current != ']') - { - if (JSON_UNLIKELY(not parse_ubjson_internal(false))) - { - return false; - } - get_ignore_noop(); - } - } - - return sax->end_array(); - } - - /*! - @return whether object creation completed - */ - bool get_ubjson_object() - { - std::pair size_and_type; - if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) - { - return false; - } - - string_t key; - if (size_and_type.first != string_t::npos) - { - if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) - { - return false; - } - - if (size_and_type.second != 0) - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) - { - return false; - } - key.clear(); - } - } - else - { - for (std::size_t i = 0; i < size_and_type.first; ++i) - { - if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - key.clear(); - } - } - } - else - { - if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) - { - return false; - } - - while (current != '}') - { - if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) - { - return false; - } - if (JSON_UNLIKELY(not parse_ubjson_internal())) - { - return false; - } - get_ignore_noop(); - key.clear(); - } - } - - return sax->end_object(); - } - - /////////////////////// - // Utility functions // - /////////////////////// - - /*! - @brief get next character from the input - - This function provides the interface to the used input adapter. It does - not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits::eof()` in that case. - - @return character read from the input - */ - int get() - { - ++chars_read; - return (current = ia->get_character()); - } - - /*! - @return character read from the input after ignoring all 'N' entries - */ - int get_ignore_noop() - { - do - { - get(); - } - while (current == 'N'); - - return current; - } - - /* - @brief read a number from the input - - @tparam NumberType the type of the number - @param[in] format the current format (for diagnostics) - @param[out] result number of type @a NumberType - - @return whether conversion completed - - @note This function needs to respect the system's endianess, because - bytes in CBOR, MessagePack, and UBJSON are stored in network order - (big endian) and therefore need reordering on little endian systems. - */ - template - bool get_number(const input_format_t format, NumberType& result) - { - // step 1: read input into array with system's byte order - std::array vec; - for (std::size_t i = 0; i < sizeof(NumberType); ++i) - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) - { - return false; - } - - // reverse byte order prior to conversion if necessary - if (is_little_endian && !InputIsLittleEndian) - { - vec[sizeof(NumberType) - i - 1] = static_cast(current); - } - else - { - vec[i] = static_cast(current); // LCOV_EXCL_LINE - } - } - - // step 2: convert array into number of type T and return - std::memcpy(&result, vec.data(), sizeof(NumberType)); - return true; - } - - /*! - @brief create a string by reading characters from the input - - @tparam NumberType the type of the number - @param[in] format the current format (for diagnostics) - @param[in] len number of characters to read - @param[out] result string created by reading @a len bytes - - @return whether string creation completed - - @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref unexpect_eof() detects the end of - the input before we run out of string memory. - */ - template - bool get_string(const input_format_t format, - const NumberType len, - string_t& result) - { - bool success = true; - std::generate_n(std::back_inserter(result), len, [this, &success, &format]() - { - get(); - if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) - { - success = false; - } - return static_cast(current); - }); - return success; - } - - /*! - @param[in] format the current format (for diagnostics) - @param[in] context further context information (for diagnostics) - @return whether the last read character is not EOF - */ - bool unexpect_eof(const input_format_t format, const char* context) const - { - if (JSON_UNLIKELY(current == std::char_traits::eof())) - { - return sax->parse_error(chars_read, "", - parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); - } - return true; - } - - /*! - @return a string representation of the last read byte - */ - std::string get_token_string() const - { - char cr[3]; - (std::snprintf)(cr, 3, "%.2hhX", static_cast(current)); - return std::string{cr}; - } - - /*! - @param[in] format the current format - @param[in] detail a detailed error message - @param[in] context further contect information - @return a message string to use in the parse_error exceptions - */ - std::string exception_message(const input_format_t format, - const std::string& detail, - const std::string& context) const - { - std::string error_msg = "syntax error while parsing "; - - switch (format) - { - case input_format_t::cbor: - error_msg += "CBOR"; - break; - - case input_format_t::msgpack: - error_msg += "MessagePack"; - break; - - case input_format_t::ubjson: - error_msg += "UBJSON"; - break; - - case input_format_t::bson: - error_msg += "BSON"; - break; - - // LCOV_EXCL_START - default: - assert(false); - // LCOV_EXCL_STOP - } - - return error_msg + " " + context + ": " + detail; - } - - private: - /// input adapter - input_adapter_t ia = nullptr; - - /// the current character - int current = std::char_traits::eof(); - - /// the number of characters read - std::size_t chars_read = 0; - - /// whether we can assume little endianess - const bool is_little_endian = little_endianess(); - - /// the SAX parser - json_sax_t* sax = nullptr; -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // reverse -#include // array -#include // uint8_t, uint16_t, uint32_t, uint64_t -#include // memcpy -#include // numeric_limits - -// #include - -// #include - - -namespace nlohmann -{ -namespace detail -{ -/////////////////// -// binary writer // -/////////////////// - -/*! -@brief serialization to CBOR and MessagePack values -*/ -template -class binary_writer -{ - using string_t = typename BasicJsonType::string_t; - - public: - /*! - @brief create a binary writer - - @param[in] adapter output adapter to write to - */ - explicit binary_writer(output_adapter_t adapter) : oa(adapter) - { - assert(oa); - } - - /*! - @param[in] j JSON value to serialize - @pre j.type() == value_t::object - */ - void write_bson(const BasicJsonType& j) - { - switch (j.type()) - { - case value_t::object: - { - write_bson_object(*j.m_value.object); - break; - } - - default: - { - JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); - } - } - } - - /*! - @param[in] j JSON value to serialize - */ - void write_cbor(const BasicJsonType& j) - { - switch (j.type()) - { - case value_t::null: - { - oa->write_character(to_char_type(0xF6)); - break; - } - - case value_t::boolean: - { - oa->write_character(j.m_value.boolean - ? to_char_type(0xF5) - : to_char_type(0xF4)); - break; - } - - case value_t::number_integer: - { - if (j.m_value.number_integer >= 0) - { - // CBOR does not differentiate between positive signed - // integers and unsigned integers. Therefore, we used the - // code from the value_t::number_unsigned case here. - if (j.m_value.number_integer <= 0x17) - { - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x18)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x19)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x1A)); - write_number(static_cast(j.m_value.number_integer)); - } - else - { - oa->write_character(to_char_type(0x1B)); - write_number(static_cast(j.m_value.number_integer)); - } - } - else - { - // The conversions below encode the sign in the first - // byte, and the value is converted to a positive number. - const auto positive_number = -1 - j.m_value.number_integer; - if (j.m_value.number_integer >= -24) - { - write_number(static_cast(0x20 + positive_number)); - } - else if (positive_number <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x38)); - write_number(static_cast(positive_number)); - } - else if (positive_number <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x39)); - write_number(static_cast(positive_number)); - } - else if (positive_number <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x3A)); - write_number(static_cast(positive_number)); - } - else - { - oa->write_character(to_char_type(0x3B)); - write_number(static_cast(positive_number)); - } - } - break; - } - - case value_t::number_unsigned: - { - if (j.m_value.number_unsigned <= 0x17) - { - write_number(static_cast(j.m_value.number_unsigned)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x18)); - write_number(static_cast(j.m_value.number_unsigned)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x19)); - write_number(static_cast(j.m_value.number_unsigned)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x1A)); - write_number(static_cast(j.m_value.number_unsigned)); - } - else - { - oa->write_character(to_char_type(0x1B)); - write_number(static_cast(j.m_value.number_unsigned)); - } - break; - } - - case value_t::number_float: - { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - break; - } - - case value_t::string: - { - // step 1: write control byte and the string length - const auto N = j.m_value.string->size(); - if (N <= 0x17) - { - write_number(static_cast(0x60 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x78)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x79)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x7A)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x7B)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP - - // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); - break; - } - - case value_t::array: - { - // step 1: write control byte and the array size - const auto N = j.m_value.array->size(); - if (N <= 0x17) - { - write_number(static_cast(0x80 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x98)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x99)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x9A)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0x9B)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP - - // step 2: write each element - for (const auto& el : *j.m_value.array) - { - write_cbor(el); - } - break; - } - - case value_t::object: - { - // step 1: write control byte and the object size - const auto N = j.m_value.object->size(); - if (N <= 0x17) - { - write_number(static_cast(0xA0 + N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0xB8)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0xB9)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0xBA)); - write_number(static_cast(N)); - } - // LCOV_EXCL_START - else if (N <= (std::numeric_limits::max)()) - { - oa->write_character(to_char_type(0xBB)); - write_number(static_cast(N)); - } - // LCOV_EXCL_STOP - - // step 2: write each element - for (const auto& el : *j.m_value.object) - { - write_cbor(el.first); - write_cbor(el.second); - } - break; - } - - default: - break; - } - } - - /*! - @param[in] j JSON value to serialize - */ - void write_msgpack(const BasicJsonType& j) - { - switch (j.type()) - { - case value_t::null: // nil - { - oa->write_character(to_char_type(0xC0)); - break; - } - - case value_t::boolean: // true and false - { - oa->write_character(j.m_value.boolean - ? to_char_type(0xC3) - : to_char_type(0xC2)); - break; - } - - case value_t::number_integer: - { - if (j.m_value.number_integer >= 0) - { - // MessagePack does not differentiate between positive - // signed integers and unsigned integers. Therefore, we used - // the code from the value_t::number_unsigned case here. - if (j.m_value.number_unsigned < 128) - { - // positive fixnum - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 8 - oa->write_character(to_char_type(0xCC)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 16 - oa->write_character(to_char_type(0xCD)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 32 - oa->write_character(to_char_type(0xCE)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 64 - oa->write_character(to_char_type(0xCF)); - write_number(static_cast(j.m_value.number_integer)); - } - } - else - { - if (j.m_value.number_integer >= -32) - { - // negative fixnum - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() and - j.m_value.number_integer <= (std::numeric_limits::max)()) - { - // int 8 - oa->write_character(to_char_type(0xD0)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() and - j.m_value.number_integer <= (std::numeric_limits::max)()) - { - // int 16 - oa->write_character(to_char_type(0xD1)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() and - j.m_value.number_integer <= (std::numeric_limits::max)()) - { - // int 32 - oa->write_character(to_char_type(0xD2)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_integer >= (std::numeric_limits::min)() and - j.m_value.number_integer <= (std::numeric_limits::max)()) - { - // int 64 - oa->write_character(to_char_type(0xD3)); - write_number(static_cast(j.m_value.number_integer)); - } - } - break; - } - - case value_t::number_unsigned: - { - if (j.m_value.number_unsigned < 128) - { - // positive fixnum - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 8 - oa->write_character(to_char_type(0xCC)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 16 - oa->write_character(to_char_type(0xCD)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 32 - oa->write_character(to_char_type(0xCE)); - write_number(static_cast(j.m_value.number_integer)); - } - else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - // uint 64 - oa->write_character(to_char_type(0xCF)); - write_number(static_cast(j.m_value.number_integer)); - } - break; - } - - case value_t::number_float: - { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - break; - } - - case value_t::string: - { - // step 1: write control byte and the string length - const auto N = j.m_value.string->size(); - if (N <= 31) - { - // fixstr - write_number(static_cast(0xA0 | N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // str 8 - oa->write_character(to_char_type(0xD9)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // str 16 - oa->write_character(to_char_type(0xDA)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // str 32 - oa->write_character(to_char_type(0xDB)); - write_number(static_cast(N)); - } - - // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); - break; - } - - case value_t::array: - { - // step 1: write control byte and the array size - const auto N = j.m_value.array->size(); - if (N <= 15) - { - // fixarray - write_number(static_cast(0x90 | N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 16 - oa->write_character(to_char_type(0xDC)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // array 32 - oa->write_character(to_char_type(0xDD)); - write_number(static_cast(N)); - } - - // step 2: write each element - for (const auto& el : *j.m_value.array) - { - write_msgpack(el); - } - break; - } - - case value_t::object: - { - // step 1: write control byte and the object size - const auto N = j.m_value.object->size(); - if (N <= 15) - { - // fixmap - write_number(static_cast(0x80 | (N & 0xF))); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 16 - oa->write_character(to_char_type(0xDE)); - write_number(static_cast(N)); - } - else if (N <= (std::numeric_limits::max)()) - { - // map 32 - oa->write_character(to_char_type(0xDF)); - write_number(static_cast(N)); - } - - // step 2: write each element - for (const auto& el : *j.m_value.object) - { - write_msgpack(el.first); - write_msgpack(el.second); - } - break; - } - - default: - break; - } - } - - /*! - @param[in] j JSON value to serialize - @param[in] use_count whether to use '#' prefixes (optimized format) - @param[in] use_type whether to use '$' prefixes (optimized format) - @param[in] add_prefix whether prefixes need to be used for this value - */ - void write_ubjson(const BasicJsonType& j, const bool use_count, - const bool use_type, const bool add_prefix = true) - { - switch (j.type()) - { - case value_t::null: - { - if (add_prefix) - { - oa->write_character(to_char_type('Z')); - } - break; - } - - case value_t::boolean: - { - if (add_prefix) - { - oa->write_character(j.m_value.boolean - ? to_char_type('T') - : to_char_type('F')); - } - break; - } - - case value_t::number_integer: - { - write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); - break; - } - - case value_t::number_unsigned: - { - write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); - break; - } - - case value_t::number_float: - { - write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); - break; - } - - case value_t::string: - { - if (add_prefix) - { - oa->write_character(to_char_type('S')); - } - write_number_with_ubjson_prefix(j.m_value.string->size(), true); - oa->write_characters( - reinterpret_cast(j.m_value.string->c_str()), - j.m_value.string->size()); - break; - } - - case value_t::array: - { - if (add_prefix) - { - oa->write_character(to_char_type('[')); - } - - bool prefix_required = true; - if (use_type and not j.m_value.array->empty()) - { - assert(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); - const bool same_prefix = std::all_of(j.begin() + 1, j.end(), - [this, first_prefix](const BasicJsonType & v) - { - return ubjson_prefix(v) == first_prefix; - }); - - if (same_prefix) - { - prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); - } - } - - if (use_count) - { - oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.array->size(), true); - } - - for (const auto& el : *j.m_value.array) - { - write_ubjson(el, use_count, use_type, prefix_required); - } - - if (not use_count) - { - oa->write_character(to_char_type(']')); - } - - break; - } - - case value_t::object: - { - if (add_prefix) - { - oa->write_character(to_char_type('{')); - } - - bool prefix_required = true; - if (use_type and not j.m_value.object->empty()) - { - assert(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); - const bool same_prefix = std::all_of(j.begin(), j.end(), - [this, first_prefix](const BasicJsonType & v) - { - return ubjson_prefix(v) == first_prefix; - }); - - if (same_prefix) - { - prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); - } - } - - if (use_count) - { - oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.object->size(), true); - } - - for (const auto& el : *j.m_value.object) - { - write_number_with_ubjson_prefix(el.first.size(), true); - oa->write_characters( - reinterpret_cast(el.first.c_str()), - el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required); - } - - if (not use_count) - { - oa->write_character(to_char_type('}')); - } - - break; - } - - default: - break; - } - } - - private: - ////////// - // BSON // - ////////// - - /*! - @return The size of a BSON document entry header, including the id marker - and the entry name size (and its null-terminator). - */ - static std::size_t calc_bson_entry_header_size(const string_t& name) - { - const auto it = name.find(static_cast(0)); - if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) - { - JSON_THROW(out_of_range::create(409, - "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); - } - - return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; - } - - /*! - @brief Writes the given @a element_type and @a name to the output adapter - */ - void write_bson_entry_header(const string_t& name, - const std::uint8_t element_type) - { - oa->write_character(to_char_type(element_type)); // boolean - oa->write_characters( - reinterpret_cast(name.c_str()), - name.size() + 1u); - } - - /*! - @brief Writes a BSON element with key @a name and boolean value @a value - */ - void write_bson_boolean(const string_t& name, - const bool value) - { - write_bson_entry_header(name, 0x08); - oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); - } - - /*! - @brief Writes a BSON element with key @a name and double value @a value - */ - void write_bson_double(const string_t& name, - const double value) - { - write_bson_entry_header(name, 0x01); - write_number(value); - } - - /*! - @return The size of the BSON-encoded string in @a value - */ - static std::size_t calc_bson_string_size(const string_t& value) - { - return sizeof(std::int32_t) + value.size() + 1ul; - } - - /*! - @brief Writes a BSON element with key @a name and string value @a value - */ - void write_bson_string(const string_t& name, - const string_t& value) - { - write_bson_entry_header(name, 0x02); - - write_number(static_cast(value.size() + 1ul)); - oa->write_characters( - reinterpret_cast(value.c_str()), - value.size() + 1); - } - - /*! - @brief Writes a BSON element with key @a name and null value - */ - void write_bson_null(const string_t& name) - { - write_bson_entry_header(name, 0x0A); - } - - /*! - @return The size of the BSON-encoded integer @a value - */ - static std::size_t calc_bson_integer_size(const std::int64_t value) - { - if ((std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)()) - { - return sizeof(std::int32_t); - } - else - { - return sizeof(std::int64_t); - } - } - - /*! - @brief Writes a BSON element with key @a name and integer @a value - */ - void write_bson_integer(const string_t& name, - const std::int64_t value) - { - if ((std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)()) - { - write_bson_entry_header(name, 0x10); // int32 - write_number(static_cast(value)); - } - else - { - write_bson_entry_header(name, 0x12); // int64 - write_number(static_cast(value)); - } - } - - /*! - @return The size of the BSON-encoded unsigned integer in @a j - */ - static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept - { - return (value <= static_cast((std::numeric_limits::max)())) - ? sizeof(std::int32_t) - : sizeof(std::int64_t); - } - - /*! - @brief Writes a BSON element with key @a name and unsigned @a value - */ - void write_bson_unsigned(const string_t& name, - const std::uint64_t value) - { - if (value <= static_cast((std::numeric_limits::max)())) - { - write_bson_entry_header(name, 0x10 /* int32 */); - write_number(static_cast(value)); - } - else if (value <= static_cast((std::numeric_limits::max)())) - { - write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(value)); - } - else - { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); - } - } - - /*! - @brief Writes a BSON element with key @a name and object @a value - */ - void write_bson_object_entry(const string_t& name, - const typename BasicJsonType::object_t& value) - { - write_bson_entry_header(name, 0x03); // object - write_bson_object(value); - } - - /*! - @return The size of the BSON-encoded array @a value - */ - static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) - { - std::size_t embedded_document_size = 0ul; - std::size_t array_index = 0ul; - - for (const auto& el : value) - { - embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); - } - - return sizeof(std::int32_t) + embedded_document_size + 1ul; - } - - /*! - @brief Writes a BSON element with key @a name and array @a value - */ - void write_bson_array(const string_t& name, - const typename BasicJsonType::array_t& value) - { - write_bson_entry_header(name, 0x04); // array - write_number(static_cast(calc_bson_array_size(value))); - - std::size_t array_index = 0ul; - - for (const auto& el : value) - { - write_bson_element(std::to_string(array_index++), el); - } - - oa->write_character(to_char_type(0x00)); - } - - /*! - @brief Calculates the size necessary to serialize the JSON value @a j with its @a name - @return The calculated size for the BSON document entry for @a j with the given @a name. - */ - static std::size_t calc_bson_element_size(const string_t& name, - const BasicJsonType& j) - { - const auto header_size = calc_bson_entry_header_size(name); - switch (j.type()) - { - case value_t::object: - return header_size + calc_bson_object_size(*j.m_value.object); - - case value_t::array: - return header_size + calc_bson_array_size(*j.m_value.array); - - case value_t::boolean: - return header_size + 1ul; - - case value_t::number_float: - return header_size + 8ul; - - case value_t::number_integer: - return header_size + calc_bson_integer_size(j.m_value.number_integer); - - case value_t::number_unsigned: - return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); - - case value_t::string: - return header_size + calc_bson_string_size(*j.m_value.string); - - case value_t::null: - return header_size + 0ul; - - // LCOV_EXCL_START - default: - assert(false); - return 0ul; - // LCOV_EXCL_STOP - }; - } - - /*! - @brief Serializes the JSON value @a j to BSON and associates it with the - key @a name. - @param name The name to associate with the JSON entity @a j within the - current BSON document - @return The size of the BSON entry - */ - void write_bson_element(const string_t& name, - const BasicJsonType& j) - { - switch (j.type()) - { - case value_t::object: - return write_bson_object_entry(name, *j.m_value.object); - - case value_t::array: - return write_bson_array(name, *j.m_value.array); - - case value_t::boolean: - return write_bson_boolean(name, j.m_value.boolean); - - case value_t::number_float: - return write_bson_double(name, j.m_value.number_float); - - case value_t::number_integer: - return write_bson_integer(name, j.m_value.number_integer); - - case value_t::number_unsigned: - return write_bson_unsigned(name, j.m_value.number_unsigned); - - case value_t::string: - return write_bson_string(name, *j.m_value.string); - - case value_t::null: - return write_bson_null(name); - - // LCOV_EXCL_START - default: - assert(false); - return; - // LCOV_EXCL_STOP - }; - } - - /*! - @brief Calculates the size of the BSON serialization of the given - JSON-object @a j. - @param[in] j JSON value to serialize - @pre j.type() == value_t::object - */ - static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) - { - std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, - [](size_t result, const typename BasicJsonType::object_t::value_type & el) - { - return result += calc_bson_element_size(el.first, el.second); - }); - - return sizeof(std::int32_t) + document_size + 1ul; - } - - /*! - @param[in] j JSON value to serialize - @pre j.type() == value_t::object - */ - void write_bson_object(const typename BasicJsonType::object_t& value) - { - write_number(static_cast(calc_bson_object_size(value))); - - for (const auto& el : value) - { - write_bson_element(el.first, el.second); - } - - oa->write_character(to_char_type(0x00)); - } - - ////////// - // CBOR // - ////////// - - static constexpr CharType get_cbor_float_prefix(float /*unused*/) - { - return to_char_type(0xFA); // Single-Precision Float - } - - static constexpr CharType get_cbor_float_prefix(double /*unused*/) - { - return to_char_type(0xFB); // Double-Precision Float - } - - ///////////// - // MsgPack // - ///////////// - - static constexpr CharType get_msgpack_float_prefix(float /*unused*/) - { - return to_char_type(0xCA); // float 32 - } - - static constexpr CharType get_msgpack_float_prefix(double /*unused*/) - { - return to_char_type(0xCB); // float 64 - } - - //////////// - // UBJSON // - //////////// - - // UBJSON: write number (floating point) - template::value, int>::type = 0> - void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) - { - if (add_prefix) - { - oa->write_character(get_ubjson_float_prefix(n)); - } - write_number(n); - } - - // UBJSON: write number (unsigned integer) - template::value, int>::type = 0> - void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) - { - if (n <= static_cast((std::numeric_limits::max)())) - { - if (add_prefix) - { - oa->write_character(to_char_type('i')); // int8 - } - write_number(static_cast(n)); - } - else if (n <= (std::numeric_limits::max)()) - { - if (add_prefix) - { - oa->write_character(to_char_type('U')); // uint8 - } - write_number(static_cast(n)); - } - else if (n <= static_cast((std::numeric_limits::max)())) - { - if (add_prefix) - { - oa->write_character(to_char_type('I')); // int16 - } - write_number(static_cast(n)); - } - else if (n <= static_cast((std::numeric_limits::max)())) - { - if (add_prefix) - { - oa->write_character(to_char_type('l')); // int32 - } - write_number(static_cast(n)); - } - else if (n <= static_cast((std::numeric_limits::max)())) - { - if (add_prefix) - { - oa->write_character(to_char_type('L')); // int64 - } - write_number(static_cast(n)); - } - else - { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); - } - } - - // UBJSON: write number (signed integer) - template::value and - not std::is_floating_point::value, int>::type = 0> - void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) - { - if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) - { - if (add_prefix) - { - oa->write_character(to_char_type('i')); // int8 - } - write_number(static_cast(n)); - } - else if (static_cast((std::numeric_limits::min)()) <= n and n <= static_cast((std::numeric_limits::max)())) - { - if (add_prefix) - { - oa->write_character(to_char_type('U')); // uint8 - } - write_number(static_cast(n)); - } - else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) - { - if (add_prefix) - { - oa->write_character(to_char_type('I')); // int16 - } - write_number(static_cast(n)); - } - else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) - { - if (add_prefix) - { - oa->write_character(to_char_type('l')); // int32 - } - write_number(static_cast(n)); - } - else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) - { - if (add_prefix) - { - oa->write_character(to_char_type('L')); // int64 - } - write_number(static_cast(n)); - } - // LCOV_EXCL_START - else - { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); - } - // LCOV_EXCL_STOP - } - - /*! - @brief determine the type prefix of container values - - @note This function does not need to be 100% accurate when it comes to - integer limits. In case a number exceeds the limits of int64_t, - this will be detected by a later call to function - write_number_with_ubjson_prefix. Therefore, we return 'L' for any - value that does not fit the previous limits. - */ - CharType ubjson_prefix(const BasicJsonType& j) const noexcept - { - switch (j.type()) - { - case value_t::null: - return 'Z'; - - case value_t::boolean: - return j.m_value.boolean ? 'T' : 'F'; - - case value_t::number_integer: - { - if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) - { - return 'i'; - } - if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) - { - return 'U'; - } - if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) - { - return 'I'; - } - if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) - { - return 'l'; - } - // no check and assume int64_t (see note above) - return 'L'; - } - - case value_t::number_unsigned: - { - if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - return 'i'; - } - if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - return 'U'; - } - if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - return 'I'; - } - if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) - { - return 'l'; - } - // no check and assume int64_t (see note above) - return 'L'; - } - - case value_t::number_float: - return get_ubjson_float_prefix(j.m_value.number_float); - - case value_t::string: - return 'S'; - - case value_t::array: - return '['; - - case value_t::object: - return '{'; - - default: // discarded values - return 'N'; - } - } - - static constexpr CharType get_ubjson_float_prefix(float /*unused*/) - { - return 'd'; // float 32 - } - - static constexpr CharType get_ubjson_float_prefix(double /*unused*/) - { - return 'D'; // float 64 - } - - /////////////////////// - // Utility functions // - /////////////////////// - - /* - @brief write a number to output input - @param[in] n number of type @a NumberType - @tparam NumberType the type of the number - @tparam OutputIsLittleEndian Set to true if output data is - required to be little endian - - @note This function needs to respect the system's endianess, because bytes - in CBOR, MessagePack, and UBJSON are stored in network order (big - endian) and therefore need reordering on little endian systems. - */ - template - void write_number(const NumberType n) - { - // step 1: write number to array of length NumberType - std::array vec; - std::memcpy(vec.data(), &n, sizeof(NumberType)); - - // step 2: write array to output (with possible reordering) - if (is_little_endian and not OutputIsLittleEndian) - { - // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); - } - - oa->write_characters(vec.data(), sizeof(NumberType)); - } - - public: - // The following to_char_type functions are implement the conversion - // between uint8_t and CharType. In case CharType is not unsigned, - // such a conversion is required to allow values greater than 128. - // See for a discussion. - template < typename C = CharType, - enable_if_t < std::is_signed::value and std::is_signed::value > * = nullptr > - static constexpr CharType to_char_type(std::uint8_t x) noexcept - { - return *reinterpret_cast(&x); - } - - template < typename C = CharType, - enable_if_t < std::is_signed::value and std::is_unsigned::value > * = nullptr > - static CharType to_char_type(std::uint8_t x) noexcept - { - static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); - static_assert(std::is_pod::value, "CharType must be POD"); - CharType result; - std::memcpy(&result, &x, sizeof(x)); - return result; - } - - template::value>* = nullptr> - static constexpr CharType to_char_type(std::uint8_t x) noexcept - { - return x; - } - - template < typename InputCharType, typename C = CharType, - enable_if_t < - std::is_signed::value and - std::is_signed::value and - std::is_same::type>::value - > * = nullptr > - static constexpr CharType to_char_type(InputCharType x) noexcept - { - return x; - } - - private: - /// whether we can assume little endianess - const bool is_little_endian = binary_reader::little_endianess(); - - /// the output - output_adapter_t oa = nullptr; -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // reverse, remove, fill, find, none_of -#include // array -#include // assert -#include // and, or -#include // localeconv, lconv -#include // labs, isfinite, isnan, signbit -#include // size_t, ptrdiff_t -#include // uint8_t -#include // snprintf -#include // numeric_limits -#include // string -#include // is_same - -// #include - -// #include - - -#include // assert -#include // or, and, not -#include // signbit, isfinite -#include // intN_t, uintN_t -#include // memcpy, memmove - -namespace nlohmann -{ -namespace detail -{ - -/*! -@brief implements the Grisu2 algorithm for binary to decimal floating-point -conversion. - -This implementation is a slightly modified version of the reference -implementation which may be obtained from -http://florian.loitsch.com/publications (bench.tar.gz). - -The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. - -For a detailed description of the algorithm see: - -[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with - Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming - Language Design and Implementation, PLDI 2010 -[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", - Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language - Design and Implementation, PLDI 1996 -*/ -namespace dtoa_impl -{ - -template -Target reinterpret_bits(const Source source) -{ - static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); - - Target target; - std::memcpy(&target, &source, sizeof(Source)); - return target; -} - -struct diyfp // f * 2^e -{ - static constexpr int kPrecision = 64; // = q - - uint64_t f = 0; - int e = 0; - - constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} - - /*! - @brief returns x - y - @pre x.e == y.e and x.f >= y.f - */ - static diyfp sub(const diyfp& x, const diyfp& y) noexcept - { - assert(x.e == y.e); - assert(x.f >= y.f); - - return {x.f - y.f, x.e}; - } - - /*! - @brief returns x * y - @note The result is rounded. (Only the upper q bits are returned.) - */ - static diyfp mul(const diyfp& x, const diyfp& y) noexcept - { - static_assert(kPrecision == 64, "internal error"); - - // Computes: - // f = round((x.f * y.f) / 2^q) - // e = x.e + y.e + q - - // Emulate the 64-bit * 64-bit multiplication: - // - // p = u * v - // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) - // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) - // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) - // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) - // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) - // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) - // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) - // - // (Since Q might be larger than 2^32 - 1) - // - // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) - // - // (Q_hi + H does not overflow a 64-bit int) - // - // = p_lo + 2^64 p_hi - - const uint64_t u_lo = x.f & 0xFFFFFFFF; - const uint64_t u_hi = x.f >> 32; - const uint64_t v_lo = y.f & 0xFFFFFFFF; - const uint64_t v_hi = y.f >> 32; - - const uint64_t p0 = u_lo * v_lo; - const uint64_t p1 = u_lo * v_hi; - const uint64_t p2 = u_hi * v_lo; - const uint64_t p3 = u_hi * v_hi; - - const uint64_t p0_hi = p0 >> 32; - const uint64_t p1_lo = p1 & 0xFFFFFFFF; - const uint64_t p1_hi = p1 >> 32; - const uint64_t p2_lo = p2 & 0xFFFFFFFF; - const uint64_t p2_hi = p2 >> 32; - - uint64_t Q = p0_hi + p1_lo + p2_lo; - - // The full product might now be computed as - // - // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) - // p_lo = p0_lo + (Q << 32) - // - // But in this particular case here, the full p_lo is not required. - // Effectively we only need to add the highest bit in p_lo to p_hi (and - // Q_hi + 1 does not overflow). - - Q += uint64_t{1} << (64 - 32 - 1); // round, ties up - - const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); - - return {h, x.e + y.e + 64}; - } - - /*! - @brief normalize x such that the significand is >= 2^(q-1) - @pre x.f != 0 - */ - static diyfp normalize(diyfp x) noexcept - { - assert(x.f != 0); - - while ((x.f >> 63) == 0) - { - x.f <<= 1; - x.e--; - } - - return x; - } - - /*! - @brief normalize x such that the result has the exponent E - @pre e >= x.e and the upper e - x.e bits of x.f must be zero. - */ - static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept - { - const int delta = x.e - target_exponent; - - assert(delta >= 0); - assert(((x.f << delta) >> delta) == x.f); - - return {x.f << delta, target_exponent}; - } -}; - -struct boundaries -{ - diyfp w; - diyfp minus; - diyfp plus; -}; - -/*! -Compute the (normalized) diyfp representing the input number 'value' and its -boundaries. - -@pre value must be finite and positive -*/ -template -boundaries compute_boundaries(FloatType value) -{ - assert(std::isfinite(value)); - assert(value > 0); - - // Convert the IEEE representation into a diyfp. - // - // If v is denormal: - // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) - // If v is normalized: - // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) - - static_assert(std::numeric_limits::is_iec559, - "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); - - constexpr int kPrecision = std::numeric_limits::digits; // = p (includes the hidden bit) - constexpr int kBias = std::numeric_limits::max_exponent - 1 + (kPrecision - 1); - constexpr int kMinExp = 1 - kBias; - constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) - - using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; - - const uint64_t bits = reinterpret_bits(value); - const uint64_t E = bits >> (kPrecision - 1); - const uint64_t F = bits & (kHiddenBit - 1); - - const bool is_denormal = (E == 0); - const diyfp v = is_denormal - ? diyfp(F, kMinExp) - : diyfp(F + kHiddenBit, static_cast(E) - kBias); - - // Compute the boundaries m- and m+ of the floating-point value - // v = f * 2^e. - // - // Determine v- and v+, the floating-point predecessor and successor if v, - // respectively. - // - // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) - // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) - // - // v+ = v + 2^e - // - // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ - // between m- and m+ round to v, regardless of how the input rounding - // algorithm breaks ties. - // - // ---+-------------+-------------+-------------+-------------+--- (A) - // v- m- v m+ v+ - // - // -----------------+------+------+-------------+-------------+--- (B) - // v- m- v m+ v+ - - const bool lower_boundary_is_closer = (F == 0 and E > 1); - const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); - const diyfp m_minus = lower_boundary_is_closer - ? diyfp(4 * v.f - 1, v.e - 2) // (B) - : diyfp(2 * v.f - 1, v.e - 1); // (A) - - // Determine the normalized w+ = m+. - const diyfp w_plus = diyfp::normalize(m_plus); - - // Determine w- = m- such that e_(w-) = e_(w+). - const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); - - return {diyfp::normalize(v), w_minus, w_plus}; -} - -// Given normalized diyfp w, Grisu needs to find a (normalized) cached -// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies -// within a certain range [alpha, gamma] (Definition 3.2 from [1]) -// -// alpha <= e = e_c + e_w + q <= gamma -// -// or -// -// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q -// <= f_c * f_w * 2^gamma -// -// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies -// -// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma -// -// or -// -// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) -// -// The choice of (alpha,gamma) determines the size of the table and the form of -// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well -// in practice: -// -// The idea is to cut the number c * w = f * 2^e into two parts, which can be -// processed independently: An integral part p1, and a fractional part p2: -// -// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e -// = (f div 2^-e) + (f mod 2^-e) * 2^e -// = p1 + p2 * 2^e -// -// The conversion of p1 into decimal form requires a series of divisions and -// modulos by (a power of) 10. These operations are faster for 32-bit than for -// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be -// achieved by choosing -// -// -e >= 32 or e <= -32 := gamma -// -// In order to convert the fractional part -// -// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... -// -// into decimal form, the fraction is repeatedly multiplied by 10 and the digits -// d[-i] are extracted in order: -// -// (10 * p2) div 2^-e = d[-1] -// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... -// -// The multiplication by 10 must not overflow. It is sufficient to choose -// -// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. -// -// Since p2 = f mod 2^-e < 2^-e, -// -// -e <= 60 or e >= -60 := alpha - -constexpr int kAlpha = -60; -constexpr int kGamma = -32; - -struct cached_power // c = f * 2^e ~= 10^k -{ - uint64_t f; - int e; - int k; -}; - -/*! -For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached -power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c -satisfies (Definition 3.2 from [1]) - - alpha <= e_c + e + q <= gamma. -*/ -inline cached_power get_cached_power_for_binary_exponent(int e) -{ - // Now - // - // alpha <= e_c + e + q <= gamma (1) - // ==> f_c * 2^alpha <= c * 2^e * 2^q - // - // and since the c's are normalized, 2^(q-1) <= f_c, - // - // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) - // ==> 2^(alpha - e - 1) <= c - // - // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as - // - // k = ceil( log_10( 2^(alpha - e - 1) ) ) - // = ceil( (alpha - e - 1) * log_10(2) ) - // - // From the paper: - // "In theory the result of the procedure could be wrong since c is rounded, - // and the computation itself is approximated [...]. In practice, however, - // this simple function is sufficient." - // - // For IEEE double precision floating-point numbers converted into - // normalized diyfp's w = f * 2^e, with q = 64, - // - // e >= -1022 (min IEEE exponent) - // -52 (p - 1) - // -52 (p - 1, possibly normalize denormal IEEE numbers) - // -11 (normalize the diyfp) - // = -1137 - // - // and - // - // e <= +1023 (max IEEE exponent) - // -52 (p - 1) - // -11 (normalize the diyfp) - // = 960 - // - // This binary exponent range [-1137,960] results in a decimal exponent - // range [-307,324]. One does not need to store a cached power for each - // k in this range. For each such k it suffices to find a cached power - // such that the exponent of the product lies in [alpha,gamma]. - // This implies that the difference of the decimal exponents of adjacent - // table entries must be less than or equal to - // - // floor( (gamma - alpha) * log_10(2) ) = 8. - // - // (A smaller distance gamma-alpha would require a larger table.) - - // NB: - // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. - - constexpr int kCachedPowersSize = 79; - constexpr int kCachedPowersMinDecExp = -300; - constexpr int kCachedPowersDecStep = 8; - - static constexpr cached_power kCachedPowers[] = - { - { 0xAB70FE17C79AC6CA, -1060, -300 }, - { 0xFF77B1FCBEBCDC4F, -1034, -292 }, - { 0xBE5691EF416BD60C, -1007, -284 }, - { 0x8DD01FAD907FFC3C, -980, -276 }, - { 0xD3515C2831559A83, -954, -268 }, - { 0x9D71AC8FADA6C9B5, -927, -260 }, - { 0xEA9C227723EE8BCB, -901, -252 }, - { 0xAECC49914078536D, -874, -244 }, - { 0x823C12795DB6CE57, -847, -236 }, - { 0xC21094364DFB5637, -821, -228 }, - { 0x9096EA6F3848984F, -794, -220 }, - { 0xD77485CB25823AC7, -768, -212 }, - { 0xA086CFCD97BF97F4, -741, -204 }, - { 0xEF340A98172AACE5, -715, -196 }, - { 0xB23867FB2A35B28E, -688, -188 }, - { 0x84C8D4DFD2C63F3B, -661, -180 }, - { 0xC5DD44271AD3CDBA, -635, -172 }, - { 0x936B9FCEBB25C996, -608, -164 }, - { 0xDBAC6C247D62A584, -582, -156 }, - { 0xA3AB66580D5FDAF6, -555, -148 }, - { 0xF3E2F893DEC3F126, -529, -140 }, - { 0xB5B5ADA8AAFF80B8, -502, -132 }, - { 0x87625F056C7C4A8B, -475, -124 }, - { 0xC9BCFF6034C13053, -449, -116 }, - { 0x964E858C91BA2655, -422, -108 }, - { 0xDFF9772470297EBD, -396, -100 }, - { 0xA6DFBD9FB8E5B88F, -369, -92 }, - { 0xF8A95FCF88747D94, -343, -84 }, - { 0xB94470938FA89BCF, -316, -76 }, - { 0x8A08F0F8BF0F156B, -289, -68 }, - { 0xCDB02555653131B6, -263, -60 }, - { 0x993FE2C6D07B7FAC, -236, -52 }, - { 0xE45C10C42A2B3B06, -210, -44 }, - { 0xAA242499697392D3, -183, -36 }, - { 0xFD87B5F28300CA0E, -157, -28 }, - { 0xBCE5086492111AEB, -130, -20 }, - { 0x8CBCCC096F5088CC, -103, -12 }, - { 0xD1B71758E219652C, -77, -4 }, - { 0x9C40000000000000, -50, 4 }, - { 0xE8D4A51000000000, -24, 12 }, - { 0xAD78EBC5AC620000, 3, 20 }, - { 0x813F3978F8940984, 30, 28 }, - { 0xC097CE7BC90715B3, 56, 36 }, - { 0x8F7E32CE7BEA5C70, 83, 44 }, - { 0xD5D238A4ABE98068, 109, 52 }, - { 0x9F4F2726179A2245, 136, 60 }, - { 0xED63A231D4C4FB27, 162, 68 }, - { 0xB0DE65388CC8ADA8, 189, 76 }, - { 0x83C7088E1AAB65DB, 216, 84 }, - { 0xC45D1DF942711D9A, 242, 92 }, - { 0x924D692CA61BE758, 269, 100 }, - { 0xDA01EE641A708DEA, 295, 108 }, - { 0xA26DA3999AEF774A, 322, 116 }, - { 0xF209787BB47D6B85, 348, 124 }, - { 0xB454E4A179DD1877, 375, 132 }, - { 0x865B86925B9BC5C2, 402, 140 }, - { 0xC83553C5C8965D3D, 428, 148 }, - { 0x952AB45CFA97A0B3, 455, 156 }, - { 0xDE469FBD99A05FE3, 481, 164 }, - { 0xA59BC234DB398C25, 508, 172 }, - { 0xF6C69A72A3989F5C, 534, 180 }, - { 0xB7DCBF5354E9BECE, 561, 188 }, - { 0x88FCF317F22241E2, 588, 196 }, - { 0xCC20CE9BD35C78A5, 614, 204 }, - { 0x98165AF37B2153DF, 641, 212 }, - { 0xE2A0B5DC971F303A, 667, 220 }, - { 0xA8D9D1535CE3B396, 694, 228 }, - { 0xFB9B7CD9A4A7443C, 720, 236 }, - { 0xBB764C4CA7A44410, 747, 244 }, - { 0x8BAB8EEFB6409C1A, 774, 252 }, - { 0xD01FEF10A657842C, 800, 260 }, - { 0x9B10A4E5E9913129, 827, 268 }, - { 0xE7109BFBA19C0C9D, 853, 276 }, - { 0xAC2820D9623BF429, 880, 284 }, - { 0x80444B5E7AA7CF85, 907, 292 }, - { 0xBF21E44003ACDD2D, 933, 300 }, - { 0x8E679C2F5E44FF8F, 960, 308 }, - { 0xD433179D9C8CB841, 986, 316 }, - { 0x9E19DB92B4E31BA9, 1013, 324 }, - }; - - // This computation gives exactly the same results for k as - // k = ceil((kAlpha - e - 1) * 0.30102999566398114) - // for |e| <= 1500, but doesn't require floating-point operations. - // NB: log_10(2) ~= 78913 / 2^18 - assert(e >= -1500); - assert(e <= 1500); - const int f = kAlpha - e - 1; - const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); - - const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; - assert(index >= 0); - assert(index < kCachedPowersSize); - static_cast(kCachedPowersSize); // Fix warning. - - const cached_power cached = kCachedPowers[index]; - assert(kAlpha <= cached.e + e + 64); - assert(kGamma >= cached.e + e + 64); - - return cached; -} - -/*! -For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. -For n == 0, returns 1 and sets pow10 := 1. -*/ -inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) -{ - // LCOV_EXCL_START - if (n >= 1000000000) - { - pow10 = 1000000000; - return 10; - } - // LCOV_EXCL_STOP - else if (n >= 100000000) - { - pow10 = 100000000; - return 9; - } - else if (n >= 10000000) - { - pow10 = 10000000; - return 8; - } - else if (n >= 1000000) - { - pow10 = 1000000; - return 7; - } - else if (n >= 100000) - { - pow10 = 100000; - return 6; - } - else if (n >= 10000) - { - pow10 = 10000; - return 5; - } - else if (n >= 1000) - { - pow10 = 1000; - return 4; - } - else if (n >= 100) - { - pow10 = 100; - return 3; - } - else if (n >= 10) - { - pow10 = 10; - return 2; - } - else - { - pow10 = 1; - return 1; - } -} - -inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, - uint64_t rest, uint64_t ten_k) -{ - assert(len >= 1); - assert(dist <= delta); - assert(rest <= delta); - assert(ten_k > 0); - - // <--------------------------- delta ----> - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // ten_k - // <------> - // <---- rest ----> - // --------------[------------------+----+--------------]-------------- - // w V - // = buf * 10^k - // - // ten_k represents a unit-in-the-last-place in the decimal representation - // stored in buf. - // Decrement buf by ten_k while this takes buf closer to w. - - // The tests are written in this order to avoid overflow in unsigned - // integer arithmetic. - - while (rest < dist - and delta - rest >= ten_k - and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) - { - assert(buf[len - 1] != '0'); - buf[len - 1]--; - rest += ten_k; - } -} - -/*! -Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. -M- and M+ must be normalized and share the same exponent -60 <= e <= -32. -*/ -inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, - diyfp M_minus, diyfp w, diyfp M_plus) -{ - static_assert(kAlpha >= -60, "internal error"); - static_assert(kGamma <= -32, "internal error"); - - // Generates the digits (and the exponent) of a decimal floating-point - // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's - // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. - // - // <--------------------------- delta ----> - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // Grisu2 generates the digits of M+ from left to right and stops as soon as - // V is in [M-,M+]. - - assert(M_plus.e >= kAlpha); - assert(M_plus.e <= kGamma); - - uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) - uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) - - // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): - // - // M+ = f * 2^e - // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e - // = ((p1 ) * 2^-e + (p2 )) * 2^e - // = p1 + p2 * 2^e - - const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); - - auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) - uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e - - // 1) - // - // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] - - assert(p1 > 0); - - uint32_t pow10; - const int k = find_largest_pow10(p1, pow10); - - // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) - // - // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) - // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) - // - // M+ = p1 + p2 * 2^e - // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e - // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e - // = d[k-1] * 10^(k-1) + ( rest) * 2^e - // - // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) - // - // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] - // - // but stop as soon as - // - // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e - - int n = k; - while (n > 0) - { - // Invariants: - // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) - // pow10 = 10^(n-1) <= p1 < 10^n - // - const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) - const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) - // - // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e - // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) - // - assert(d <= 9); - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) - // - p1 = r; - n--; - // - // M+ = buffer * 10^n + (p1 + p2 * 2^e) - // pow10 = 10^n - // - - // Now check if enough digits have been generated. - // Compute - // - // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e - // - // Note: - // Since rest and delta share the same exponent e, it suffices to - // compare the significands. - const uint64_t rest = (uint64_t{p1} << -one.e) + p2; - if (rest <= delta) - { - // V = buffer * 10^n, with M- <= V <= M+. - - decimal_exponent += n; - - // We may now just stop. But instead look if the buffer could be - // decremented to bring V closer to w. - // - // pow10 = 10^n is now 1 ulp in the decimal representation V. - // The rounding procedure works with diyfp's with an implicit - // exponent of e. - // - // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e - // - const uint64_t ten_n = uint64_t{pow10} << -one.e; - grisu2_round(buffer, length, dist, delta, rest, ten_n); - - return; - } - - pow10 /= 10; - // - // pow10 = 10^(n-1) <= p1 < 10^n - // Invariants restored. - } - - // 2) - // - // The digits of the integral part have been generated: - // - // M+ = d[k-1]...d[1]d[0] + p2 * 2^e - // = buffer + p2 * 2^e - // - // Now generate the digits of the fractional part p2 * 2^e. - // - // Note: - // No decimal point is generated: the exponent is adjusted instead. - // - // p2 actually represents the fraction - // - // p2 * 2^e - // = p2 / 2^-e - // = d[-1] / 10^1 + d[-2] / 10^2 + ... - // - // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) - // - // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m - // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) - // - // using - // - // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) - // = ( d) * 2^-e + ( r) - // - // or - // 10^m * p2 * 2^e = d + r * 2^e - // - // i.e. - // - // M+ = buffer + p2 * 2^e - // = buffer + 10^-m * (d + r * 2^e) - // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e - // - // and stop as soon as 10^-m * r * 2^e <= delta * 2^e - - assert(p2 > delta); - - int m = 0; - for (;;) - { - // Invariant: - // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e - // = buffer * 10^-m + 10^-m * (p2 ) * 2^e - // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e - // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e - // - assert(p2 <= UINT64_MAX / 10); - p2 *= 10; - const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e - const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e - // - // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e - // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) - // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - assert(d <= 9); - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - p2 = r; - m++; - // - // M+ = buffer * 10^-m + 10^-m * p2 * 2^e - // Invariant restored. - - // Check if enough digits have been generated. - // - // 10^-m * p2 * 2^e <= delta * 2^e - // p2 * 2^e <= 10^m * delta * 2^e - // p2 <= 10^m * delta - delta *= 10; - dist *= 10; - if (p2 <= delta) - { - break; - } - } - - // V = buffer * 10^-m, with M- <= V <= M+. - - decimal_exponent -= m; - - // 1 ulp in the decimal representation is now 10^-m. - // Since delta and dist are now scaled by 10^m, we need to do the - // same with ulp in order to keep the units in sync. - // - // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e - // - const uint64_t ten_m = one.f; - grisu2_round(buffer, length, dist, delta, p2, ten_m); - - // By construction this algorithm generates the shortest possible decimal - // number (Loitsch, Theorem 6.2) which rounds back to w. - // For an input number of precision p, at least - // - // N = 1 + ceil(p * log_10(2)) - // - // decimal digits are sufficient to identify all binary floating-point - // numbers (Matula, "In-and-Out conversions"). - // This implies that the algorithm does not produce more than N decimal - // digits. - // - // N = 17 for p = 53 (IEEE double precision) - // N = 9 for p = 24 (IEEE single precision) -} - -/*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. -*/ -inline void grisu2(char* buf, int& len, int& decimal_exponent, - diyfp m_minus, diyfp v, diyfp m_plus) -{ - assert(m_plus.e == m_minus.e); - assert(m_plus.e == v.e); - - // --------(-----------------------+-----------------------)-------- (A) - // m- v m+ - // - // --------------------(-----------+-----------------------)-------- (B) - // m- v m+ - // - // First scale v (and m- and m+) such that the exponent is in the range - // [alpha, gamma]. - - const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); - - const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k - - // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] - const diyfp w = diyfp::mul(v, c_minus_k); - const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); - const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); - - // ----(---+---)---------------(---+---)---------------(---+---)---- - // w- w w+ - // = c*m- = c*v = c*m+ - // - // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and - // w+ are now off by a small amount. - // In fact: - // - // w - v * 10^k < 1 ulp - // - // To account for this inaccuracy, add resp. subtract 1 ulp. - // - // --------+---[---------------(---+---)---------------]---+-------- - // w- M- w M+ w+ - // - // Now any number in [M-, M+] (bounds included) will round to w when input, - // regardless of how the input rounding algorithm breaks ties. - // - // And digit_gen generates the shortest possible such number in [M-, M+]. - // Note that this does not mean that Grisu2 always generates the shortest - // possible number in the interval (m-, m+). - const diyfp M_minus(w_minus.f + 1, w_minus.e); - const diyfp M_plus (w_plus.f - 1, w_plus.e ); - - decimal_exponent = -cached.k; // = -(-k) = k - - grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); -} - -/*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. -*/ -template -void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) -{ - static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, - "internal error: not enough precision"); - - assert(std::isfinite(value)); - assert(value > 0); - - // If the neighbors (and boundaries) of 'value' are always computed for double-precision - // numbers, all float's can be recovered using strtod (and strtof). However, the resulting - // decimal representations are not exactly "short". - // - // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) - // says "value is converted to a string as if by std::sprintf in the default ("C") locale" - // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' - // does. - // On the other hand, the documentation for 'std::to_chars' requires that "parsing the - // representation using the corresponding std::from_chars function recovers value exactly". That - // indicates that single precision floating-point numbers should be recovered using - // 'std::strtof'. - // - // NB: If the neighbors are computed for single-precision numbers, there is a single float - // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision - // value is off by 1 ulp. -#if 0 - const boundaries w = compute_boundaries(static_cast(value)); -#else - const boundaries w = compute_boundaries(value); -#endif - - grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); -} - -/*! -@brief appends a decimal representation of e to buf -@return a pointer to the element following the exponent. -@pre -1000 < e < 1000 -*/ -inline char* append_exponent(char* buf, int e) -{ - assert(e > -1000); - assert(e < 1000); - - if (e < 0) - { - e = -e; - *buf++ = '-'; - } - else - { - *buf++ = '+'; - } - - auto k = static_cast(e); - if (k < 10) - { - // Always print at least two digits in the exponent. - // This is for compatibility with printf("%g"). - *buf++ = '0'; - *buf++ = static_cast('0' + k); - } - else if (k < 100) - { - *buf++ = static_cast('0' + k / 10); - k %= 10; - *buf++ = static_cast('0' + k); - } - else - { - *buf++ = static_cast('0' + k / 100); - k %= 100; - *buf++ = static_cast('0' + k / 10); - k %= 10; - *buf++ = static_cast('0' + k); - } - - return buf; -} - -/*! -@brief prettify v = buf * 10^decimal_exponent - -If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point -notation. Otherwise it will be printed in exponential notation. - -@pre min_exp < 0 -@pre max_exp > 0 -*/ -inline char* format_buffer(char* buf, int len, int decimal_exponent, - int min_exp, int max_exp) -{ - assert(min_exp < 0); - assert(max_exp > 0); - - const int k = len; - const int n = len + decimal_exponent; - - // v = buf * 10^(n-k) - // k is the length of the buffer (number of decimal digits) - // n is the position of the decimal point relative to the start of the buffer. - - if (k <= n and n <= max_exp) - { - // digits[000] - // len <= max_exp + 2 - - std::memset(buf + k, '0', static_cast(n - k)); - // Make it look like a floating-point number (#362, #378) - buf[n + 0] = '.'; - buf[n + 1] = '0'; - return buf + (n + 2); - } - - if (0 < n and n <= max_exp) - { - // dig.its - // len <= max_digits10 + 1 - - assert(k > n); - - std::memmove(buf + (n + 1), buf + n, static_cast(k - n)); - buf[n] = '.'; - return buf + (k + 1); - } - - if (min_exp < n and n <= 0) - { - // 0.[000]digits - // len <= 2 + (-min_exp - 1) + max_digits10 - - std::memmove(buf + (2 + -n), buf, static_cast(k)); - buf[0] = '0'; - buf[1] = '.'; - std::memset(buf + 2, '0', static_cast(-n)); - return buf + (2 + (-n) + k); - } - - if (k == 1) - { - // dE+123 - // len <= 1 + 5 - - buf += 1; - } - else - { - // d.igitsE+123 - // len <= max_digits10 + 1 + 5 - - std::memmove(buf + 2, buf + 1, static_cast(k - 1)); - buf[1] = '.'; - buf += 1 + k; - } - - *buf++ = 'e'; - return append_exponent(buf, n - 1); -} - -} // namespace dtoa_impl - -/*! -@brief generates a decimal representation of the floating-point number value in [first, last). - -The format of the resulting decimal representation is similar to printf's %g -format. Returns an iterator pointing past-the-end of the decimal representation. - -@note The input number must be finite, i.e. NaN's and Inf's are not supported. -@note The buffer must be large enough. -@note The result is NOT null-terminated. -*/ -template -char* to_chars(char* first, const char* last, FloatType value) -{ - static_cast(last); // maybe unused - fix warning - assert(std::isfinite(value)); - - // Use signbit(value) instead of (value < 0) since signbit works for -0. - if (std::signbit(value)) - { - value = -value; - *first++ = '-'; - } - - if (value == 0) // +-0 - { - *first++ = '0'; - // Make it look like a floating-point number (#362, #378) - *first++ = '.'; - *first++ = '0'; - return first; - } - - assert(last - first >= std::numeric_limits::max_digits10); - - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e. the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - - assert(len <= std::numeric_limits::max_digits10); - - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - // Use digits10 here to increase compatibility with version 2. - constexpr int kMaxExp = std::numeric_limits::digits10; - - assert(last - first >= kMaxExp + 2); - assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); - assert(last - first >= std::numeric_limits::max_digits10 + 6); - - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); -} - -} // namespace detail -} // namespace nlohmann - -// #include - -// #include - -// #include - -// #include - -// #include - - -namespace nlohmann -{ -namespace detail -{ -/////////////////// -// serialization // -/////////////////// - -/// how to treat decoding errors -enum class error_handler_t -{ - strict, ///< throw a type_error exception in case of invalid UTF-8 - replace, ///< replace invalid UTF-8 sequences with U+FFFD - ignore ///< ignore invalid UTF-8 sequences -}; - -template -class serializer -{ - using string_t = typename BasicJsonType::string_t; - using number_float_t = typename BasicJsonType::number_float_t; - using number_integer_t = typename BasicJsonType::number_integer_t; - using number_unsigned_t = typename BasicJsonType::number_unsigned_t; - static constexpr uint8_t UTF8_ACCEPT = 0; - static constexpr uint8_t UTF8_REJECT = 1; - - public: - /*! - @param[in] s output stream to serialize to - @param[in] ichar indentation character to use - @param[in] error_handler_ how to react on decoding errors - */ - serializer(output_adapter_t s, const char ichar, - error_handler_t error_handler_ = error_handler_t::strict) - : o(std::move(s)) - , loc(std::localeconv()) - , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) - , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) - , indent_char(ichar) - , indent_string(512, indent_char) - , error_handler(error_handler_) - {} - - // delete because of pointer members - serializer(const serializer&) = delete; - serializer& operator=(const serializer&) = delete; - serializer(serializer&&) = delete; - serializer& operator=(serializer&&) = delete; - ~serializer() = default; - - /*! - @brief internal implementation of the serialization function - - This function is called by the public member function dump and organizes - the serialization internally. The indentation level is propagated as - additional parameter. In case of arrays and objects, the function is - called recursively. - - - strings and object keys are escaped using `escape_string()` - - integer numbers are converted implicitly via `operator<<` - - floating-point numbers are converted to a string using `"%g"` format - - @param[in] val value to serialize - @param[in] pretty_print whether the output shall be pretty-printed - @param[in] indent_step the indent level - @param[in] current_indent the current indent level (only used internally) - */ - void dump(const BasicJsonType& val, const bool pretty_print, - const bool ensure_ascii, - const unsigned int indent_step, - const unsigned int current_indent = 0) - { - switch (val.m_type) - { - case value_t::object: - { - if (val.m_value.object->empty()) - { - o->write_characters("{}", 2); - return; - } - - if (pretty_print) - { - o->write_characters("{\n", 2); - - // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } - - // first n-1 elements - auto i = val.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) - { - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); - } - - // last element - assert(i != val.m_value.object->cend()); - assert(std::next(i) == val.m_value.object->cend()); - o->write_characters(indent_string.c_str(), new_indent); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\": ", 3); - dump(i->second, true, ensure_ascii, indent_step, new_indent); - - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character('}'); - } - else - { - o->write_character('{'); - - // first n-1 elements - auto i = val.m_value.object->cbegin(); - for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) - { - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); - } - - // last element - assert(i != val.m_value.object->cend()); - assert(std::next(i) == val.m_value.object->cend()); - o->write_character('\"'); - dump_escaped(i->first, ensure_ascii); - o->write_characters("\":", 2); - dump(i->second, false, ensure_ascii, indent_step, current_indent); - - o->write_character('}'); - } - - return; - } - - case value_t::array: - { - if (val.m_value.array->empty()) - { - o->write_characters("[]", 2); - return; - } - - if (pretty_print) - { - o->write_characters("[\n", 2); - - // variable to hold indentation for recursive calls - const auto new_indent = current_indent + indent_step; - if (JSON_UNLIKELY(indent_string.size() < new_indent)) - { - indent_string.resize(indent_string.size() * 2, ' '); - } - - // first n-1 elements - for (auto i = val.m_value.array->cbegin(); - i != val.m_value.array->cend() - 1; ++i) - { - o->write_characters(indent_string.c_str(), new_indent); - dump(*i, true, ensure_ascii, indent_step, new_indent); - o->write_characters(",\n", 2); - } - - // last element - assert(not val.m_value.array->empty()); - o->write_characters(indent_string.c_str(), new_indent); - dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); - - o->write_character('\n'); - o->write_characters(indent_string.c_str(), current_indent); - o->write_character(']'); - } - else - { - o->write_character('['); - - // first n-1 elements - for (auto i = val.m_value.array->cbegin(); - i != val.m_value.array->cend() - 1; ++i) - { - dump(*i, false, ensure_ascii, indent_step, current_indent); - o->write_character(','); - } - - // last element - assert(not val.m_value.array->empty()); - dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); - - o->write_character(']'); - } - - return; - } - - case value_t::string: - { - o->write_character('\"'); - dump_escaped(*val.m_value.string, ensure_ascii); - o->write_character('\"'); - return; - } - - case value_t::boolean: - { - if (val.m_value.boolean) - { - o->write_characters("true", 4); - } - else - { - o->write_characters("false", 5); - } - return; - } - - case value_t::number_integer: - { - dump_integer(val.m_value.number_integer); - return; - } - - case value_t::number_unsigned: - { - dump_integer(val.m_value.number_unsigned); - return; - } - - case value_t::number_float: - { - dump_float(val.m_value.number_float); - return; - } - - case value_t::discarded: - { - o->write_characters("", 11); - return; - } - - case value_t::null: - { - o->write_characters("null", 4); - return; - } - } - } - - private: - /*! - @brief dump escaped string - - Escape a string by replacing certain special characters by a sequence of an - escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. The escaped string is written to output stream @a o. - - @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences - - @complexity Linear in the length of string @a s. - */ - void dump_escaped(const string_t& s, const bool ensure_ascii) - { - uint32_t codepoint; - uint8_t state = UTF8_ACCEPT; - std::size_t bytes = 0; // number of bytes written to string_buffer - - // number of bytes written at the point of the last valid byte - std::size_t bytes_after_last_accept = 0; - std::size_t undumped_chars = 0; - - for (std::size_t i = 0; i < s.size(); ++i) - { - const auto byte = static_cast(s[i]); - - switch (decode(state, codepoint, byte)) - { - case UTF8_ACCEPT: // decode found a new code point - { - switch (codepoint) - { - case 0x08: // backspace - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'b'; - break; - } - - case 0x09: // horizontal tab - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 't'; - break; - } - - case 0x0A: // newline - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'n'; - break; - } - - case 0x0C: // formfeed - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'f'; - break; - } - - case 0x0D: // carriage return - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'r'; - break; - } - - case 0x22: // quotation mark - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\"'; - break; - } - - case 0x5C: // reverse solidus - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = '\\'; - break; - } - - default: - { - // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) - { - if (codepoint <= 0xFFFF) - { - (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", - static_cast(codepoint)); - bytes += 6; - } - else - { - (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", - static_cast(0xD7C0 + (codepoint >> 10)), - static_cast(0xDC00 + (codepoint & 0x3FF))); - bytes += 12; - } - } - else - { - // copy byte to buffer (all previous bytes - // been copied have in default case above) - string_buffer[bytes++] = s[i]; - } - break; - } - } - - // write buffer and reset index; there must be 13 bytes - // left, as this is the maximal number of bytes to be - // written ("\uxxxx\uxxxx\0") for one code point - if (string_buffer.size() - bytes < 13) - { - o->write_characters(string_buffer.data(), bytes); - bytes = 0; - } - - // remember the byte position of this accept - bytes_after_last_accept = bytes; - undumped_chars = 0; - break; - } - - case UTF8_REJECT: // decode found invalid UTF-8 byte - { - switch (error_handler) - { - case error_handler_t::strict: - { - std::string sn(3, '\0'); - (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); - JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); - } - - case error_handler_t::ignore: - case error_handler_t::replace: - { - // in case we saw this character the first time, we - // would like to read it again, because the byte - // may be OK for itself, but just not OK for the - // previous sequence - if (undumped_chars > 0) - { - --i; - } - - // reset length buffer to the last accepted index; - // thus removing/ignoring the invalid characters - bytes = bytes_after_last_accept; - - if (error_handler == error_handler_t::replace) - { - // add a replacement character - if (ensure_ascii) - { - string_buffer[bytes++] = '\\'; - string_buffer[bytes++] = 'u'; - string_buffer[bytes++] = 'f'; - string_buffer[bytes++] = 'f'; - string_buffer[bytes++] = 'f'; - string_buffer[bytes++] = 'd'; - } - else - { - string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); - string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); - string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); - } - bytes_after_last_accept = bytes; - } - - undumped_chars = 0; - - // continue processing the string - state = UTF8_ACCEPT; - break; - } - } - break; - } - - default: // decode found yet incomplete multi-byte code point - { - if (not ensure_ascii) - { - // code point will not be escaped - copy byte to buffer - string_buffer[bytes++] = s[i]; - } - ++undumped_chars; - break; - } - } - } - - // we finished processing the string - if (JSON_LIKELY(state == UTF8_ACCEPT)) - { - // write buffer - if (bytes > 0) - { - o->write_characters(string_buffer.data(), bytes); - } - } - else - { - // we finish reading, but do not accept: string was incomplete - switch (error_handler) - { - case error_handler_t::strict: - { - std::string sn(3, '\0'); - (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast(s.back())); - JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); - } - - case error_handler_t::ignore: - { - // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); - break; - } - - case error_handler_t::replace: - { - // write all accepted bytes - o->write_characters(string_buffer.data(), bytes_after_last_accept); - // add a replacement character - if (ensure_ascii) - { - o->write_characters("\\ufffd", 6); - } - else - { - o->write_characters("\xEF\xBF\xBD", 3); - } - break; - } - } - } - } - - /*! - @brief dump an integer - - Dump a given integer to output stream @a o. Works internally with - @a number_buffer. - - @param[in] x integer number (signed or unsigned) to dump - @tparam NumberType either @a number_integer_t or @a number_unsigned_t - */ - template::value or - std::is_same::value, - int> = 0> - void dump_integer(NumberType x) - { - // special case for "0" - if (x == 0) - { - o->write_character('0'); - return; - } - - const bool is_negative = std::is_same::value and not (x >= 0); // see issue #755 - std::size_t i = 0; - - while (x != 0) - { - // spare 1 byte for '\0' - assert(i < number_buffer.size() - 1); - - const auto digit = std::labs(static_cast(x % 10)); - number_buffer[i++] = static_cast('0' + digit); - x /= 10; - } - - if (is_negative) - { - // make sure there is capacity for the '-' - assert(i < number_buffer.size() - 2); - number_buffer[i++] = '-'; - } - - std::reverse(number_buffer.begin(), number_buffer.begin() + i); - o->write_characters(number_buffer.data(), i); - } - - /*! - @brief dump a floating-point number - - Dump a given floating-point number to output stream @a o. Works internally - with @a number_buffer. - - @param[in] x floating-point number to dump - */ - void dump_float(number_float_t x) - { - // NaN / inf - if (not std::isfinite(x)) - { - o->write_characters("null", 4); - return; - } - - // If number_float_t is an IEEE-754 single or double precision number, - // use the Grisu2 algorithm to produce short numbers which are - // guaranteed to round-trip, using strtof and strtod, resp. - // - // NB: The test below works if == . - static constexpr bool is_ieee_single_or_double - = (std::numeric_limits::is_iec559 and std::numeric_limits::digits == 24 and std::numeric_limits::max_exponent == 128) or - (std::numeric_limits::is_iec559 and std::numeric_limits::digits == 53 and std::numeric_limits::max_exponent == 1024); - - dump_float(x, std::integral_constant()); - } - - void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) - { - char* begin = number_buffer.data(); - char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); - - o->write_characters(begin, static_cast(end - begin)); - } - - void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) - { - // get number of digits for a float -> text -> float round-trip - static constexpr auto d = std::numeric_limits::max_digits10; - - // the actual conversion - std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); - - // negative value indicates an error - assert(len > 0); - // check if buffer was large enough - assert(static_cast(len) < number_buffer.size()); - - // erase thousands separator - if (thousands_sep != '\0') - { - const auto end = std::remove(number_buffer.begin(), - number_buffer.begin() + len, thousands_sep); - std::fill(end, number_buffer.end(), '\0'); - assert((end - number_buffer.begin()) <= len); - len = (end - number_buffer.begin()); - } - - // convert decimal point to '.' - if (decimal_point != '\0' and decimal_point != '.') - { - const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); - if (dec_pos != number_buffer.end()) - { - *dec_pos = '.'; - } - } - - o->write_characters(number_buffer.data(), static_cast(len)); - - // determine if need to append ".0" - const bool value_is_int_like = - std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, - [](char c) - { - return (c == '.' or c == 'e'); - }); - - if (value_is_int_like) - { - o->write_characters(".0", 2); - } - } - - /*! - @brief check whether a string is UTF-8 encoded - - The function checks each byte of a string whether it is UTF-8 encoded. The - result of the check is stored in the @a state parameter. The function must - be called initially with state 0 (accept). State 1 means the string must - be rejected, because the current byte is not allowed. If the string is - completely processed, but the state is non-zero, the string ended - prematurely; that is, the last byte indicated more bytes should have - followed. - - @param[in,out] state the state of the decoding - @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) - @param[in] byte next byte to decode - @return new state - - @note The function has been edited: a std::array is used. - - @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann - @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ - */ - static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept - { - static const std::array utf8d = - { - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF - 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF - 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF - 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF - 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 - 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 - 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 - 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 - } - }; - - const uint8_t type = utf8d[byte]; - - codep = (state != UTF8_ACCEPT) - ? (byte & 0x3fu) | (codep << 6) - : static_cast(0xff >> type) & (byte); - - state = utf8d[256u + state * 16u + type]; - return state; - } - - private: - /// the output of the serializer - output_adapter_t o = nullptr; - - /// a (hopefully) large enough character buffer - std::array number_buffer{{}}; - - /// the locale - const std::lconv* loc = nullptr; - /// the locale's thousand separator character - const char thousands_sep = '\0'; - /// the locale's decimal point character - const char decimal_point = '\0'; - - /// string buffer - std::array string_buffer{{}}; - - /// the indentation character - const char indent_char; - /// the indentation string - string_t indent_string; - - /// error_handler how to react on decoding errors - const error_handler_t error_handler; -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include -#include - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -class json_ref -{ - public: - using value_type = BasicJsonType; - - json_ref(value_type&& value) - : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) - {} - - json_ref(const value_type& value) - : value_ref(const_cast(&value)), is_rvalue(false) - {} - - json_ref(std::initializer_list init) - : owned_value(init), value_ref(&owned_value), is_rvalue(true) - {} - - template < - class... Args, - enable_if_t::value, int> = 0 > - json_ref(Args && ... args) - : owned_value(std::forward(args)...), value_ref(&owned_value), - is_rvalue(true) {} - - // class should be movable only - json_ref(json_ref&&) = default; - json_ref(const json_ref&) = delete; - json_ref& operator=(const json_ref&) = delete; - json_ref& operator=(json_ref&&) = delete; - ~json_ref() = default; - - value_type moved_or_copied() const - { - if (is_rvalue) - { - return std::move(*value_ref); - } - return *value_ref; - } - - value_type const& operator*() const - { - return *static_cast(value_ref); - } - - value_type const* operator->() const - { - return static_cast(value_ref); - } - - private: - mutable value_type owned_value = nullptr; - value_type* value_ref = nullptr; - const bool is_rvalue; -}; -} // namespace detail -} // namespace nlohmann +// #include // #include +#include // all_of #include // assert #include // accumulate #include // string +#include // move #include // vector -// #include - // #include +// #include + // #include @@ -11755,8 +8466,7 @@ class json_pointer @return a string representation of the JSON pointer - @liveexample{The example shows the result of `to_string`., - json_pointer__to_string} + @liveexample{The example shows the result of `to_string`.,json_pointer__to_string} @since version 2.0.0 */ @@ -11776,6 +8486,249 @@ class json_pointer return to_string(); } + /*! + @brief append another JSON pointer at the end of this JSON pointer + + @param[in] ptr JSON pointer to append + @return JSON pointer with @a ptr appended + + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} + + @complexity Linear in the length of @a ptr. + + @sa @ref operator/=(std::string) to append a reference token + @sa @ref operator/=(std::size_t) to append an array index + @sa @ref operator/(const json_pointer&, const json_pointer&) for a binary operator + + @since version 3.6.0 + */ + json_pointer& operator/=(const json_pointer& ptr) + { + reference_tokens.insert(reference_tokens.end(), + ptr.reference_tokens.begin(), + ptr.reference_tokens.end()); + return *this; + } + + /*! + @brief append an unescaped reference token at the end of this JSON pointer + + @param[in] token reference token to append + @return JSON pointer with @a token appended without escaping @a token + + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} + + @complexity Amortized constant. + + @sa @ref operator/=(const json_pointer&) to append a JSON pointer + @sa @ref operator/=(std::size_t) to append an array index + @sa @ref operator/(const json_pointer&, std::size_t) for a binary operator + + @since version 3.6.0 + */ + json_pointer& operator/=(std::string token) + { + push_back(std::move(token)); + return *this; + } + + /*! + @brief append an array index at the end of this JSON pointer + + @param[in] array_index array index ot append + @return JSON pointer with @a array_index appended + + @liveexample{The example shows the usage of `operator/=`.,json_pointer__operator_add} + + @complexity Amortized constant. + + @sa @ref operator/=(const json_pointer&) to append a JSON pointer + @sa @ref operator/=(std::string) to append a reference token + @sa @ref operator/(const json_pointer&, std::string) for a binary operator + + @since version 3.6.0 + */ + json_pointer& operator/=(std::size_t array_index) + { + return *this /= std::to_string(array_index); + } + + /*! + @brief create a new JSON pointer by appending the right JSON pointer at the end of the left JSON pointer + + @param[in] lhs JSON pointer + @param[in] rhs JSON pointer + @return a new JSON pointer with @a rhs appended to @a lhs + + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} + + @complexity Linear in the length of @a lhs and @a rhs. + + @sa @ref operator/=(const json_pointer&) to append a JSON pointer + + @since version 3.6.0 + */ + friend json_pointer operator/(const json_pointer& lhs, + const json_pointer& rhs) + { + return json_pointer(lhs) /= rhs; + } + + /*! + @brief create a new JSON pointer by appending the unescaped token at the end of the JSON pointer + + @param[in] ptr JSON pointer + @param[in] token reference token + @return a new JSON pointer with unescaped @a token appended to @a ptr + + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} + + @complexity Linear in the length of @a ptr. + + @sa @ref operator/=(std::string) to append a reference token + + @since version 3.6.0 + */ + friend json_pointer operator/(const json_pointer& ptr, std::string token) + { + return json_pointer(ptr) /= std::move(token); + } + + /*! + @brief create a new JSON pointer by appending the array-index-token at the end of the JSON pointer + + @param[in] ptr JSON pointer + @param[in] array_index array index + @return a new JSON pointer with @a array_index appended to @a ptr + + @liveexample{The example shows the usage of `operator/`.,json_pointer__operator_add_binary} + + @complexity Linear in the length of @a ptr. + + @sa @ref operator/=(std::size_t) to append an array index + + @since version 3.6.0 + */ + friend json_pointer operator/(const json_pointer& ptr, std::size_t array_index) + { + return json_pointer(ptr) /= array_index; + } + + /*! + @brief returns the parent of this JSON pointer + + @return parent of this JSON pointer; in case this JSON pointer is the root, + the root itself is returned + + @complexity Linear in the length of the JSON pointer. + + @liveexample{The example shows the result of `parent_pointer` for different + JSON Pointers.,json_pointer__parent_pointer} + + @since version 3.6.0 + */ + json_pointer parent_pointer() const + { + if (empty()) + { + return *this; + } + + json_pointer res = *this; + res.pop_back(); + return res; + } + + /*! + @brief remove last reference token + + @pre not `empty()` + + @liveexample{The example shows the usage of `pop_back`.,json_pointer__pop_back} + + @complexity Constant. + + @throw out_of_range.405 if JSON pointer has no parent + + @since version 3.6.0 + */ + void pop_back() + { + if (JSON_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } + + reference_tokens.pop_back(); + } + + /*! + @brief return last reference token + + @pre not `empty()` + @return last reference token + + @liveexample{The example shows the usage of `back`.,json_pointer__back} + + @complexity Constant. + + @throw out_of_range.405 if JSON pointer has no parent + + @since version 3.6.0 + */ + const std::string& back() + { + if (JSON_UNLIKELY(empty())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } + + return reference_tokens.back(); + } + + /*! + @brief append an unescaped token at the end of the reference pointer + + @param[in] token token to add + + @complexity Amortized constant. + + @liveexample{The example shows the result of `push_back` for different + JSON Pointers.,json_pointer__push_back} + + @since version 3.6.0 + */ + void push_back(const std::string& token) + { + reference_tokens.push_back(token); + } + + /// @copydoc push_back(const std::string&) + void push_back(std::string&& token) + { + reference_tokens.push_back(std::move(token)); + } + + /*! + @brief return whether pointer points to the root document + + @return true iff the JSON pointer points to the root document + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example shows the result of `empty` for different JSON + Pointers.,json_pointer__empty} + + @since version 3.6.0 + */ + bool empty() const noexcept + { + return reference_tokens.empty(); + } + + private: /*! @param[in] s reference token to be converted into an array index @@ -11797,32 +8750,9 @@ class json_pointer return res; } - private: - /*! - @brief remove and return last reference pointer - @throw out_of_range.405 if JSON pointer has no parent - */ - std::string pop_back() - { - if (JSON_UNLIKELY(is_root())) - { - JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); - } - - auto last = reference_tokens.back(); - reference_tokens.pop_back(); - return last; - } - - /// return whether pointer points to the root document - bool is_root() const noexcept - { - return reference_tokens.empty(); - } - json_pointer top() const { - if (JSON_UNLIKELY(is_root())) + if (JSON_UNLIKELY(empty())) { JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); } @@ -11933,7 +8863,7 @@ class json_pointer std::all_of(reference_token.begin(), reference_token.end(), [](const char x) { - return (x >= '0' and x <= '9'); + return x >= '0' and x <= '9'; }); // change value to array for numbers or "-" or to object otherwise @@ -12379,12 +9309,34 @@ class json_pointer return result; } + /*! + @brief compares two JSON pointers for equality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is equal to @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ friend bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept { - return (lhs.reference_tokens == rhs.reference_tokens); + return lhs.reference_tokens == rhs.reference_tokens; } + /*! + @brief compares two JSON pointers for inequality + + @param[in] lhs JSON pointer to compare + @param[in] rhs JSON pointer to compare + @return whether @a lhs is not equal @a rhs + + @complexity Linear in the length of the JSON pointer + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + */ friend bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept { @@ -12396,59 +9348,3494 @@ class json_pointer }; } // namespace nlohmann -// #include +// #include +#include #include -// #include - -// #include +// #include namespace nlohmann { - -template -struct adl_serializer +namespace detail { - /*! - @brief convert a JSON value to any value type +template +class json_ref +{ + public: + using value_type = BasicJsonType; - This function is usually called by the `get()` function of the - @ref basic_json class (either explicit or via conversion operators). + json_ref(value_type&& value) + : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) + {} - @param[in] j JSON value to read from - @param[in,out] val value to write to - */ - template - static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( - noexcept(::nlohmann::from_json(std::forward(j), val))) - -> decltype(::nlohmann::from_json(std::forward(j), val), void()) + json_ref(const value_type& value) + : value_ref(const_cast(&value)), is_rvalue(false) + {} + + json_ref(std::initializer_list init) + : owned_value(init), value_ref(&owned_value), is_rvalue(true) + {} + + template < + class... Args, + enable_if_t::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward(args)...), value_ref(&owned_value), + is_rvalue(true) {} + + // class should be movable only + json_ref(json_ref&&) = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; + + value_type moved_or_copied() const { - ::nlohmann::from_json(std::forward(j), val); + if (is_rvalue) + { + return std::move(*value_ref); + } + return *value_ref; + } + + value_type const& operator*() const + { + return *static_cast(value_ref); + } + + value_type const* operator->() const + { + return static_cast(value_ref); + } + + private: + mutable value_type owned_value = nullptr; + value_type* value_ref = nullptr; + const bool is_rvalue; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + + +#include // reverse +#include // array +#include // uint8_t, uint16_t, uint32_t, uint64_t +#include // memcpy +#include // numeric_limits +#include // string + +// #include + +// #include + + +#include // copy +#include // size_t +#include // streamsize +#include // back_inserter +#include // shared_ptr, make_shared +#include // basic_ostream +#include // basic_string +#include // vector + +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; +}; + +/// a type to simplify interfaces +template +using output_adapter_t = std::shared_ptr>; + +/// output adapter for byte vectors +template +class output_vector_adapter : public output_adapter_protocol +{ + public: + explicit output_vector_adapter(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) override + { + v.push_back(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } + + private: + std::vector& v; +}; + +/// output adapter for output streams +template +class output_stream_adapter : public output_adapter_protocol +{ + public: + explicit output_stream_adapter(std::basic_ostream& s) noexcept + : stream(s) + {} + + void write_character(CharType c) override + { + stream.put(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + stream.write(s, static_cast(length)); + } + + private: + std::basic_ostream& stream; +}; + +/// output adapter for basic_string +template> +class output_string_adapter : public output_adapter_protocol +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override + { + str.push_back(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + str.append(s, length); + } + + private: + StringType& str; +}; + +template> +class output_adapter +{ + public: + output_adapter(std::vector& vec) + : oa(std::make_shared>(vec)) {} + + output_adapter(std::basic_ostream& s) + : oa(std::make_shared>(s)) {} + + output_adapter(StringType& s) + : oa(std::make_shared>(s)) {} + + operator output_adapter_t() + { + return oa; + } + + private: + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// + +/*! +@brief serialization to CBOR and MessagePack values +*/ +template +class binary_writer +{ + using string_t = typename BasicJsonType::string_t; + + public: + /*! + @brief create a binary writer + + @param[in] adapter output adapter to write to + */ + explicit binary_writer(output_adapter_t adapter) : oa(adapter) + { + assert(oa); } /*! - @brief convert any value type to a JSON value - - This function is usually called by the constructors of the @ref basic_json - class. - - @param[in,out] j JSON value to write to - @param[in] val value to read from + @param[in] j JSON value to serialize + @pre j.type() == value_t::object */ - template - static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( - noexcept(::nlohmann::to_json(j, std::forward(val)))) - -> decltype(::nlohmann::to_json(j, std::forward(val)), void()) + void write_bson(const BasicJsonType& j) { - ::nlohmann::to_json(j, std::forward(val)); + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); + } + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_cbor(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xF6)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_integer)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + write_number(static_cast(0x20 + positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x38)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x39)); + write_number(static_cast(positive_number)); + } + else if (positive_number <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x3A)); + write_number(static_cast(positive_number)); + } + else + { + oa->write_character(to_char_type(0x3B)); + write_number(static_cast(positive_number)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast(j.m_value.number_unsigned)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast(j.m_value.number_unsigned)); + } + break; + } + + case value_t::number_float: + { + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + write_number(static_cast(0x60 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x78)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x79)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x7B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + write_number(static_cast(0x80 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x98)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x99)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9A)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0x9B)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_cbor(el); + } + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + write_number(static_cast(0xA0 + N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB8)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xB9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBA)); + write_number(static_cast(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits::max)()) + { + oa->write_character(to_char_type(0xBB)); + write_number(static_cast(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_cbor(el.first); + write_cbor(el.second); + } + break; + } + + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_msgpack(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: // nil + { + oa->write_character(to_char_type(0xC0)); + break; + } + + case value_t::boolean: // true and false + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we used + // the code from the value_t::number_unsigned case here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 8 + oa->write_character(to_char_type(0xD0)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 16 + oa->write_character(to_char_type(0xD1)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 32 + oa->write_character(to_char_type(0xD2)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits::min)() and + j.m_value.number_integer <= (std::numeric_limits::max)()) + { + // int 64 + oa->write_character(to_char_type(0xD3)); + write_number(static_cast(j.m_value.number_integer)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast(j.m_value.number_integer)); + } + break; + } + + case value_t::number_float: + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + write_number(static_cast(0xA0 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 8 + oa->write_character(to_char_type(0xD9)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 16 + oa->write_character(to_char_type(0xDA)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // str 32 + oa->write_character(to_char_type(0xDB)); + write_number(static_cast(N)); + } + + // step 2: write the string + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + write_number(static_cast(0x90 | N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 16 + oa->write_character(to_char_type(0xDC)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // array 32 + oa->write_character(to_char_type(0xDD)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_msgpack(el); + } + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + write_number(static_cast(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 16 + oa->write_character(to_char_type(0xDE)); + write_number(static_cast(N)); + } + else if (N <= (std::numeric_limits::max)()) + { + // map 32 + oa->write_character(to_char_type(0xDF)); + write_number(static_cast(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_msgpack(el.first); + write_msgpack(el.second); + } + break; + } + + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + default: + break; + } + } + + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name) + { + const auto it = name.find(static_cast(0)); + if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, + "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number(static_cast(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + return (std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)() + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits::min)() <= value and value <= (std::numeric_limits::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number(static_cast(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number(static_cast(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast((std::numeric_limits::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const std::uint64_t value) + { + if (value <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number(static_cast(value)); + } + else if (value <= static_cast((std::numeric_limits::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number(static_cast(value)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); + } + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number(static_cast(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + default: + assert(false); + return 0ul; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + @return The size of the BSON entry + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j.m_value.number_unsigned); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + default: + assert(false); + return; + // LCOV_EXCL_STOP + } + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number(static_cast(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if (n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + } + + // UBJSON: write number (signed integer) + template::value and + not std::is_floating_point::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast(n)); + } + else if (static_cast((std::numeric_limits::min)()) <= n and n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast(n)); + } + else if ((std::numeric_limits::min)() <= n and n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast(n)); + } + // LCOV_EXCL_START + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'i'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'U'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'I'; + } + if ((std::numeric_limits::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'i'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'U'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'I'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianess, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template + void write_number(const NumberType n) + { + // step 1: write number to array of length NumberType + std::array vec; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian != OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); + } + + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed::value and std::is_signed::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed::value and std::is_unsigned::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_pod::value, "CharType must be POD"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed::value and + std::is_signed::value and + std::is_same::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + + private: + /// whether we can assume little endianess + const bool is_little_endian = binary_reader::little_endianess(); + + /// the output + output_adapter_t oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + + +#include // reverse, remove, fill, find, none_of +#include // array +#include // assert +#include // and, or +#include // localeconv, lconv +#include // labs, isfinite, isnan, signbit +#include // size_t, ptrdiff_t +#include // uint8_t +#include // snprintf +#include // numeric_limits +#include // string +#include // is_same +#include // move + +// #include + + +#include // array +#include // assert +#include // or, and, not +#include // signbit, isfinite +#include // intN_t, uintN_t +#include // memcpy, memmove +#include // numeric_limits +#include // conditional + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + std::uint64_t f = 0; + int e = 0; + + constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + assert(x.e == y.e); + assert(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; + const std::uint64_t u_hi = x.f >> 32u; + const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; + const std::uint64_t v_hi = y.f >> 32u; + + const std::uint64_t p0 = u_lo * v_lo; + const std::uint64_t p1 = u_lo * v_hi; + const std::uint64_t p2 = u_hi * v_lo; + const std::uint64_t p3 = u_hi * v_hi; + + const std::uint64_t p0_hi = p0 >> 32u; + const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; + const std::uint64_t p1_hi = p1 >> 32u; + const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; + const std::uint64_t p2_hi = p2 >> 32u; + + std::uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up + + const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + assert(x.f != 0); + + while ((x.f >> 63u) == 0) + { + x.f <<= 1u; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + assert(delta >= 0); + assert(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; } }; +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template +boundaries compute_boundaries(FloatType value) +{ + assert(std::isfinite(value)); + assert(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr std::uint64_t kHiddenBit = std::uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional::type; + + const std::uint64_t bits = reinterpret_bits(value); + const std::uint64_t E = bits >> (kPrecision - 1); + const std::uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = E == 0; + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = F == 0 and E > 1; + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + std::uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr std::array kCachedPowers = + { + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + } + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + assert(e >= -1500); + assert(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + assert(index >= 0); + assert(static_cast(index) < kCachedPowers.size()); + + const cached_power cached = kCachedPowers[static_cast(index)]; + assert(kAlpha <= cached.e + e + 64); + assert(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const std::uint32_t n, std::uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + else if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + else if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + else if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + else if (n >= 100000) + { + pow10 = 100000; + return 6; + } + else if (n >= 10000) + { + pow10 = 10000; + return 5; + } + else if (n >= 1000) + { + pow10 = 1000; + return 4; + } + else if (n >= 100) + { + pow10 = 100; + return 3; + } + else if (n >= 10) + { + pow10 = 10; + return 2; + } + else + { + pow10 = 1; + return 1; + } +} + +inline void grisu2_round(char* buf, int len, std::uint64_t dist, std::uint64_t delta, + std::uint64_t rest, std::uint64_t ten_k) +{ + assert(len >= 1); + assert(dist <= delta); + assert(rest <= delta); + assert(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + and delta - rest >= ten_k + and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) + { + assert(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + assert(M_plus.e >= kAlpha); + assert(M_plus.e <= kGamma); + + std::uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + std::uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + assert(p1 > 0); + + std::uint32_t pow10; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + assert(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + assert(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + assert(p2 <= (std::numeric_limits::max)() / 10); + p2 *= 10; + const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + assert(d <= 9); + buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const std::uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + assert(m_plus.e == m_minus.e); + assert(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, + "internal error: not enough precision"); + + assert(std::isfinite(value)); + assert(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +inline char* append_exponent(char* buf, int e) +{ + assert(e > -1000); + assert(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + else + { + *buf++ = static_cast('0' + k / 100); + k %= 100; + *buf++ = static_cast('0' + k / 10); + k %= 10; + *buf++ = static_cast('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + assert(min_exp < 0); + assert(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n and n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast(n - k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (n + 2); + } + + if (0 < n and n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + assert(k > n); + + std::memmove(buf + (n + 1), buf + n, static_cast(k - n)); + buf[n] = '.'; + return buf + (k + 1); + } + + if (min_exp < n and n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + -n), buf, static_cast(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast(-n)); + return buf + (2 + (-n) + k); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast(k - 1)); + buf[1] = '.'; + buf += 1 + k; + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast(last); // maybe unused - fix warning + assert(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + + assert(last - first >= std::numeric_limits::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + assert(len <= std::numeric_limits::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits::digits10; + + assert(last - first >= kMaxExp + 2); + assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits::max_digits10); + assert(last - first >= std::numeric_limits::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include + +// #include + +// #include + +// #include + +// #include + +// #include + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// serialization // +/////////////////// + +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + +template +class serializer +{ + using string_t = typename BasicJsonType::string_t; + using number_float_t = typename BasicJsonType::number_float_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + static constexpr std::uint8_t UTF8_ACCEPT = 0; + static constexpr std::uint8_t UTF8_REJECT = 1; + + public: + /*! + @param[in] s output stream to serialize to + @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors + */ + serializer(output_adapter_t s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) + , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} + + // delete because of pointer members + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const BasicJsonType& val, const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o->write_characters("{}", 2); + return; + } + + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + assert(i != val.m_value.object->cend()); + assert(std::next(i) == val.m_value.object->cend()); + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_character('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + assert(i != val.m_value.object->cend()); + assert(std::next(i) == val.m_value.object->cend()); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + + o->write_character('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o->write_characters("[]", 2); + return; + } + + if (pretty_print) + { + o->write_characters("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + dump(*i, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + assert(not val.m_value.array->empty()); + o->write_characters(indent_string.c_str(), new_indent); + dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character(']'); + } + else + { + o->write_character('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + assert(not val.m_value.array->empty()); + dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + + o->write_character(']'); + } + + return; + } + + case value_t::string: + { + o->write_character('\"'); + dump_escaped(*val.m_value.string, ensure_ascii); + o->write_character('\"'); + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o->write_characters("true", 4); + } + else + { + o->write_characters("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o->write_characters("", 11); + return; + } + + case value_t::null: + { + o->write_characters("null", 4); + return; + } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + } + + private: + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + @param[in] ensure_ascii whether to escape non-ASCII characters with + \uXXXX sequences + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s, const bool ensure_ascii) + { + std::uint32_t codepoint; + std::uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + const auto byte = static_cast(s[i]); + + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point + { + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } + + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } + + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } + + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } + + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } + + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } + + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } + + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast(codepoint)); + bytes += 6; + } + else + { + (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast(0xD7C0u + (codepoint >> 10u)), + static_cast(0xDC00u + (codepoint & 0x3FFu))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + break; + } + + default: // decode found yet incomplete multi-byte code point + { + if (not ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; + break; + } + } + } + + // we finished processing the string + if (JSON_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE + } + } + } + + /*! + @brief count digits + + Count the number of decimal (base 10) digits for an input unsigned integer. + + @param[in] x unsigned integer number to count its digits + @return number of decimal digits + */ + inline unsigned int count_digits(number_unsigned_t x) noexcept + { + unsigned int n_digits = 1; + for (;;) + { + if (x < 10) + { + return n_digits; + } + if (x < 100) + { + return n_digits + 1; + } + if (x < 1000) + { + return n_digits + 2; + } + if (x < 10000) + { + return n_digits + 3; + } + x = x / 10000u; + n_digits += 4; + } + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template::value or + std::is_same::value, + int> = 0> + void dump_integer(NumberType x) + { + static constexpr std::array, 100> digits_to_99 + { + { + {{'0', '0'}}, {{'0', '1'}}, {{'0', '2'}}, {{'0', '3'}}, {{'0', '4'}}, {{'0', '5'}}, {{'0', '6'}}, {{'0', '7'}}, {{'0', '8'}}, {{'0', '9'}}, + {{'1', '0'}}, {{'1', '1'}}, {{'1', '2'}}, {{'1', '3'}}, {{'1', '4'}}, {{'1', '5'}}, {{'1', '6'}}, {{'1', '7'}}, {{'1', '8'}}, {{'1', '9'}}, + {{'2', '0'}}, {{'2', '1'}}, {{'2', '2'}}, {{'2', '3'}}, {{'2', '4'}}, {{'2', '5'}}, {{'2', '6'}}, {{'2', '7'}}, {{'2', '8'}}, {{'2', '9'}}, + {{'3', '0'}}, {{'3', '1'}}, {{'3', '2'}}, {{'3', '3'}}, {{'3', '4'}}, {{'3', '5'}}, {{'3', '6'}}, {{'3', '7'}}, {{'3', '8'}}, {{'3', '9'}}, + {{'4', '0'}}, {{'4', '1'}}, {{'4', '2'}}, {{'4', '3'}}, {{'4', '4'}}, {{'4', '5'}}, {{'4', '6'}}, {{'4', '7'}}, {{'4', '8'}}, {{'4', '9'}}, + {{'5', '0'}}, {{'5', '1'}}, {{'5', '2'}}, {{'5', '3'}}, {{'5', '4'}}, {{'5', '5'}}, {{'5', '6'}}, {{'5', '7'}}, {{'5', '8'}}, {{'5', '9'}}, + {{'6', '0'}}, {{'6', '1'}}, {{'6', '2'}}, {{'6', '3'}}, {{'6', '4'}}, {{'6', '5'}}, {{'6', '6'}}, {{'6', '7'}}, {{'6', '8'}}, {{'6', '9'}}, + {{'7', '0'}}, {{'7', '1'}}, {{'7', '2'}}, {{'7', '3'}}, {{'7', '4'}}, {{'7', '5'}}, {{'7', '6'}}, {{'7', '7'}}, {{'7', '8'}}, {{'7', '9'}}, + {{'8', '0'}}, {{'8', '1'}}, {{'8', '2'}}, {{'8', '3'}}, {{'8', '4'}}, {{'8', '5'}}, {{'8', '6'}}, {{'8', '7'}}, {{'8', '8'}}, {{'8', '9'}}, + {{'9', '0'}}, {{'9', '1'}}, {{'9', '2'}}, {{'9', '3'}}, {{'9', '4'}}, {{'9', '5'}}, {{'9', '6'}}, {{'9', '7'}}, {{'9', '8'}}, {{'9', '9'}}, + } + }; + + // special case for "0" + if (x == 0) + { + o->write_character('0'); + return; + } + + // use a pointer to fill the buffer + auto buffer_ptr = number_buffer.begin(); + + const bool is_negative = std::is_same::value and not(x >= 0); // see issue #755 + number_unsigned_t abs_value; + + unsigned int n_chars; + + if (is_negative) + { + *buffer_ptr = '-'; + abs_value = static_cast(std::abs(static_cast(x))); + + // account one more byte for the minus sign + n_chars = 1 + count_digits(abs_value); + } + else + { + abs_value = static_cast(x); + n_chars = count_digits(abs_value); + } + + // spare 1 byte for '\0' + assert(n_chars < number_buffer.size() - 1); + + // jump to the end to generate the string from backward + // so we later avoid reversing the result + buffer_ptr += n_chars; + + // Fast int2ascii implementation inspired by "Fastware" talk by Andrei Alexandrescu + // See: https://www.youtube.com/watch?v=o4-CwDo2zpg + while (abs_value >= 100) + { + const auto digits_index = static_cast((abs_value % 100)); + abs_value /= 100; + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + + if (abs_value >= 10) + { + const auto digits_index = static_cast(abs_value); + *(--buffer_ptr) = digits_to_99[digits_index][1]; + *(--buffer_ptr) = digits_to_99[digits_index][0]; + } + else + { + *(--buffer_ptr) = static_cast('0' + abs_value); + } + + o->write_characters(number_buffer.data(), n_chars); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works internally + with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (not std::isfinite(x)) + { + o->write_characters("null", 4); + return; + } + + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if == . + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits::is_iec559 and std::numeric_limits::digits == 24 and std::numeric_limits::max_exponent == 128) or + (std::numeric_limits::is_iec559 and std::numeric_limits::digits == 53 and std::numeric_limits::max_exponent == 1024); + + dump_float(x, std::integral_constant()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + char* begin = number_buffer.data(); + char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits::max_digits10; + + // the actual conversion + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + + // negative value indicates an error + assert(len > 0); + // check if buffer was large enough + assert(static_cast(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(number_buffer.begin(), + number_buffer.begin() + len, thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + assert((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + if (dec_pos != number_buffer.end()) + { + *dec_pos = '.'; + } + } + + o->write_characters(number_buffer.data(), static_cast(len)); + + // determine if need to append ".0" + const bool value_is_int_like = + std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, + [](char c) + { + return c == '.' or c == 'e'; + }); + + if (value_is_int_like) + { + o->write_characters(".0", 2); + } + } + + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) + @param[in] byte next byte to decode + @return new state + + @note The function has been edited: a std::array is used. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static std::uint8_t decode(std::uint8_t& state, std::uint32_t& codep, const std::uint8_t byte) noexcept + { + static const std::array utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + const std::uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6u) + : (0xFFu >> type) & (byte); + + state = utf8d[256u + state * 16u + type]; + return state; + } + + private: + /// the output of the serializer + output_adapter_t o = nullptr; + + /// a (hopefully) large enough character buffer + std::array number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// string buffer + std::array string_buffer{{}}; + + /// the indentation character + const char indent_char; + /// the indentation string + string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; +}; +} // namespace detail } // namespace nlohmann +// #include + +// #include + /*! @brief namespace for Niels Lohmann @@ -12697,6 +13084,7 @@ class basic_json @since 2.1.0 */ + JSON_NODISCARD static basic_json meta() { basic_json result; @@ -13329,7 +13717,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.6.1")); // LCOV_EXCL_LINE } break; } @@ -13702,6 +14090,8 @@ class basic_json case value_t::discarded: m_type = value_t::discarded; break; + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } assert_invariant(); } @@ -13789,7 +14179,7 @@ class basic_json bool is_an_object = std::all_of(init.begin(), init.end(), [](const detail::json_ref& element_ref) { - return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string()); + return element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string(); }); // adjust type if type deduction is not wanted @@ -13869,6 +14259,7 @@ class basic_json @since version 1.0.0 */ + JSON_NODISCARD static basic_json array(initializer_list_t init = {}) { return basic_json(init, false, value_t::array); @@ -13912,6 +14303,7 @@ class basic_json @since version 1.0.0 */ + JSON_NODISCARD static basic_json object(initializer_list_t init = {}) { return basic_json(init, false, value_t::object); @@ -14472,7 +14864,7 @@ class basic_json */ constexpr bool is_null() const noexcept { - return (m_type == value_t::null); + return m_type == value_t::null; } /*! @@ -14494,7 +14886,7 @@ class basic_json */ constexpr bool is_boolean() const noexcept { - return (m_type == value_t::boolean); + return m_type == value_t::boolean; } /*! @@ -14553,7 +14945,7 @@ class basic_json */ constexpr bool is_number_integer() const noexcept { - return (m_type == value_t::number_integer or m_type == value_t::number_unsigned); + return m_type == value_t::number_integer or m_type == value_t::number_unsigned; } /*! @@ -14581,7 +14973,7 @@ class basic_json */ constexpr bool is_number_unsigned() const noexcept { - return (m_type == value_t::number_unsigned); + return m_type == value_t::number_unsigned; } /*! @@ -14609,7 +15001,7 @@ class basic_json */ constexpr bool is_number_float() const noexcept { - return (m_type == value_t::number_float); + return m_type == value_t::number_float; } /*! @@ -14631,7 +15023,7 @@ class basic_json */ constexpr bool is_object() const noexcept { - return (m_type == value_t::object); + return m_type == value_t::object; } /*! @@ -14653,7 +15045,7 @@ class basic_json */ constexpr bool is_array() const noexcept { - return (m_type == value_t::array); + return m_type == value_t::array; } /*! @@ -14675,7 +15067,7 @@ class basic_json */ constexpr bool is_string() const noexcept { - return (m_type == value_t::string); + return m_type == value_t::string; } /*! @@ -14702,7 +15094,7 @@ class basic_json */ constexpr bool is_discarded() const noexcept { - return (m_type == value_t::discarded); + return m_type == value_t::discarded; } /*! @@ -15236,7 +15628,7 @@ class basic_json #ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same>::value -#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 +#if defined(JSON_HAS_CPP_17) && (defined(__GNUC__) || (defined(_MSC_VER) and _MSC_VER <= 1914)) and not std::is_same::value #endif #endif @@ -16274,6 +16666,8 @@ class basic_json @liveexample{The example shows how `find()` is used.,find__key_type} + @sa @ref contains(KeyT&&) const -- checks whether a key exists + @since version 1.0.0 */ template @@ -16334,6 +16728,36 @@ class basic_json return is_object() ? m_value.object->count(std::forward(key)) : 0; } + /*! + @brief check the existence of an element in a JSON object + + Check whether an element exists in a JSON object with key equivalent to + @a key. If the element is not found or the JSON value is not an object, + false is returned. + + @note This method always returns false when executed on a JSON type + that is not an object. + + @param[in] key key value to check its existence. + + @return true if an element with specified @a key exists. If no such + element with such key is found or the JSON value is not an object, + false is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The following code shows an example for `contains()`.,contains} + + @sa @ref find(KeyT&&) -- returns an iterator to an object element + + @since version 3.6.0 + */ + template + bool contains(KeyT&& key) const + { + return is_object() and m_value.object->find(std::forward(key)) != m_value.object->end(); + } + /// @} @@ -17118,7 +17542,8 @@ class basic_json // add element to array (move semantics) m_value.array->push_back(std::move(val)); - // invalidate object + // invalidate object: mark it null so we do not call the destructor + // cppcheck-suppress accessMoved val.m_type = value_t::null; } @@ -17910,28 +18335,28 @@ class basic_json switch (lhs_type) { case value_t::array: - return (*lhs.m_value.array == *rhs.m_value.array); + return *lhs.m_value.array == *rhs.m_value.array; case value_t::object: - return (*lhs.m_value.object == *rhs.m_value.object); + return *lhs.m_value.object == *rhs.m_value.object; case value_t::null: return true; case value_t::string: - return (*lhs.m_value.string == *rhs.m_value.string); + return *lhs.m_value.string == *rhs.m_value.string; case value_t::boolean: - return (lhs.m_value.boolean == rhs.m_value.boolean); + return lhs.m_value.boolean == rhs.m_value.boolean; case value_t::number_integer: - return (lhs.m_value.number_integer == rhs.m_value.number_integer); + return lhs.m_value.number_integer == rhs.m_value.number_integer; case value_t::number_unsigned: - return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned); + return lhs.m_value.number_unsigned == rhs.m_value.number_unsigned; case value_t::number_float: - return (lhs.m_value.number_float == rhs.m_value.number_float); + return lhs.m_value.number_float == rhs.m_value.number_float; default: return false; @@ -17939,27 +18364,27 @@ class basic_json } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) { - return (static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float); + return static_cast(lhs.m_value.number_integer) == rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) { - return (lhs.m_value.number_float == static_cast(rhs.m_value.number_integer)); + return lhs.m_value.number_float == static_cast(rhs.m_value.number_integer); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) { - return (static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float); + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_float; } else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) { - return (lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned)); + return lhs.m_value.number_float == static_cast(rhs.m_value.number_unsigned); } else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) { - return (static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer); + return static_cast(lhs.m_value.number_unsigned) == rhs.m_value.number_integer; } else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) { - return (lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned)); + return lhs.m_value.number_integer == static_cast(rhs.m_value.number_unsigned); } return false; @@ -17973,7 +18398,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs == basic_json(rhs)); + return lhs == basic_json(rhs); } /*! @@ -17984,7 +18409,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) == rhs); + return basic_json(lhs) == rhs; } /*! @@ -18018,7 +18443,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs != basic_json(rhs)); + return lhs != basic_json(rhs); } /*! @@ -18029,7 +18454,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) != rhs); + return basic_json(lhs) != rhs; } /*! @@ -18068,6 +18493,8 @@ class basic_json switch (lhs_type) { case value_t::array: + // note parentheses are necessary, see + // https://github.com/nlohmann/json/issues/1530 return (*lhs.m_value.array) < (*rhs.m_value.array); case value_t::object: @@ -18134,7 +18561,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs < basic_json(rhs)); + return lhs < basic_json(rhs); } /*! @@ -18145,7 +18572,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) < rhs); + return basic_json(lhs) < rhs; } /*! @@ -18180,7 +18607,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs <= basic_json(rhs)); + return lhs <= basic_json(rhs); } /*! @@ -18191,7 +18618,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) <= rhs); + return basic_json(lhs) <= rhs; } /*! @@ -18226,7 +18653,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs > basic_json(rhs)); + return lhs > basic_json(rhs); } /*! @@ -18237,7 +18664,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) > rhs); + return basic_json(lhs) > rhs; } /*! @@ -18272,7 +18699,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept { - return (lhs >= basic_json(rhs)); + return lhs >= basic_json(rhs); } /*! @@ -18283,7 +18710,7 @@ class basic_json std::is_scalar::value, int>::type = 0> friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept { - return (basic_json(lhs) >= rhs); + return basic_json(lhs) >= rhs; } /// @} @@ -18329,8 +18756,8 @@ class basic_json friend std::ostream& operator<<(std::ostream& o, const basic_json& j) { // read width member and use it as indentation parameter if nonzero - const bool pretty_print = (o.width() > 0); - const auto indentation = (pretty_print ? o.width() : 0); + const bool pretty_print = o.width() > 0; + const auto indentation = pretty_print ? o.width() : 0; // reset width to 0 for subsequent calls to this stream o.width(0); @@ -18386,9 +18813,6 @@ class basic_json @pre The container storage is contiguous. Violating this precondition yields undefined behavior. **This precondition is enforced with an assertion.** - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** @warning There is no way to enforce all preconditions at compile-time. If the function is called with a noncompliant container and with @@ -18402,7 +18826,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return result of the deserialization + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.101 if a parse error occurs; example: `""unexpected end of input; expected string literal""` @@ -18429,6 +18855,7 @@ class basic_json @since version 2.0.3 (contiguous containers) */ + JSON_NODISCARD static basic_json parse(detail::input_adapter&& i, const parser_callback_t cb = nullptr, const bool allow_exceptions = true) @@ -18466,9 +18893,6 @@ class basic_json @pre The container storage is contiguous. Violating this precondition yields undefined behavior. **This precondition is enforced with an assertion.** - @pre Each element of the container has a size of 1 byte. Violating this - precondition yields undefined behavior. **This precondition is enforced - with a static assertion.** @warning There is no way to enforce all preconditions at compile-time. If the function is called with a noncompliant container and with @@ -18505,13 +18929,9 @@ class basic_json const bool strict = true) { assert(sax); - switch (format) - { - case input_format_t::json: - return parser(std::move(i)).sax_parse(sax, strict); - default: - return detail::binary_reader(std::move(i)).sax_parse(format, sax, strict); - } + return format == input_format_t::json + ? parser(std::move(i)).sax_parse(sax, strict) + : detail::binary_reader(std::move(i)).sax_parse(format, sax, strict); } /*! @@ -18544,7 +18964,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return result of the deserialization + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.101 in case of an unexpected token @throw parse_error.102 if to_unicode fails or surrogate error @@ -19175,7 +19597,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19200,6 +19624,7 @@ class basic_json @a strict parameter since 3.0.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_cbor(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19215,6 +19640,7 @@ class basic_json */ template::value, int> = 0> + JSON_NODISCARD static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19278,7 +19704,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19305,6 +19733,7 @@ class basic_json @a strict parameter since 3.0.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_msgpack(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19320,6 +19749,7 @@ class basic_json */ template::value, int> = 0> + JSON_NODISCARD static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19365,7 +19795,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -19389,6 +19821,7 @@ class basic_json @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0 */ + JSON_NODISCARD static basic_json from_ubjson(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19404,6 +19837,7 @@ class basic_json */ template::value, int> = 0> + JSON_NODISCARD static basic_json from_ubjson(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19454,7 +19888,9 @@ class basic_json @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) - @return deserialized JSON value + @return deserialized JSON value; in case of a parse error and + @a allow_exceptions set to `false`, the return value will be + value_t::discarded. @throw parse_error.114 if an unsupported BSON record type is encountered @@ -19472,6 +19908,7 @@ class basic_json @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the related UBJSON format */ + JSON_NODISCARD static basic_json from_bson(detail::input_adapter&& i, const bool strict = true, const bool allow_exceptions = true) @@ -19487,6 +19924,7 @@ class basic_json */ template::value, int> = 0> + JSON_NODISCARD static basic_json from_bson(A1 && a1, A2 && a2, const bool strict = true, const bool allow_exceptions = true) @@ -19822,63 +20260,59 @@ class basic_json const auto operation_add = [&result](json_pointer & ptr, basic_json val) { // adding to the root of the target document means replacing it - if (ptr.is_root()) + if (ptr.empty()) { result = val; + return; } - else + + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) { - // make sure the top element of the pointer exists - json_pointer top_pointer = ptr.top(); - if (top_pointer != ptr) + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.back(); + ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: { - result.at(top_pointer); + // use operator[] to add value + parent[last_path] = val; + break; } - // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); - basic_json& parent = result[ptr]; - - switch (parent.m_type) + case value_t::array: { - case value_t::null: - case value_t::object: + if (last_path == "-") { - // use operator[] to add value - parent[last_path] = val; - break; + // special case: append to back + parent.push_back(val); } - - case value_t::array: + else { - if (last_path == "-") + const auto idx = json_pointer::array_index(last_path); + if (JSON_UNLIKELY(static_cast(idx) > parent.size())) { - // special case: append to back - parent.push_back(val); + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } - else - { - const auto idx = json_pointer::array_index(last_path); - if (JSON_UNLIKELY(static_cast(idx) > parent.size())) - { - // avoid undefined behavior - JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); - } - // default case: insert add offset - parent.insert(parent.begin() + static_cast(idx), val); - } - break; + // default case: insert add offset + parent.insert(parent.begin() + static_cast(idx), val); } - - // LCOV_EXCL_START - default: - { - // if there exists a parent it cannot be primitive - assert(false); - } - // LCOV_EXCL_STOP + break; } + + // if there exists a parent it cannot be primitive + default: // LCOV_EXCL_LINE + assert(false); // LCOV_EXCL_LINE } }; @@ -19886,7 +20320,8 @@ class basic_json const auto operation_remove = [&result](json_pointer & ptr) { // get reference to parent of JSON pointer ptr - const auto last_path = ptr.pop_back(); + const auto last_path = ptr.back(); + ptr.pop_back(); basic_json& parent = result.at(ptr); // remove child @@ -20033,7 +20468,7 @@ class basic_json break; } - case patch_operations::invalid: + default: { // op must be "add", "remove", "replace", "move", "copy", or // "test" @@ -20078,6 +20513,7 @@ class basic_json @since version 2.0.0 */ + JSON_NODISCARD static basic_json diff(const basic_json& source, const basic_json& target, const std::string& path = "") { @@ -20097,106 +20533,105 @@ class basic_json { {"op", "replace"}, {"path", path}, {"value", target} }); + return result; } - else + + switch (source.type()) { - switch (source.type()) + case value_t::array: { - case value_t::array: + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() and i < target.size()) { - // first pass: traverse common elements - std::size_t i = 0; - while (i < source.size() and i < target.size()) - { - // recursive call to compare array values at index i - auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - ++i; - } - - // i now reached the end of at least one array - // in a second pass, traverse the remaining elements - - // remove my remaining elements - const auto end_index = static_cast(result.size()); - while (i < source.size()) - { - // add operations in reverse order to avoid invalid - // indices - result.insert(result.begin() + end_index, object( - { - {"op", "remove"}, - {"path", path + "/" + std::to_string(i)} - })); - ++i; - } - - // add other remaining elements - while (i < target.size()) - { - result.push_back( - { - {"op", "add"}, - {"path", path + "/" + std::to_string(i)}, - {"value", target[i]} - }); - ++i; - } - - break; + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; } - case value_t::object: + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast(result.size()); + while (i < source.size()) { - // first pass: traverse this object's elements - for (auto it = source.cbegin(); it != source.cend(); ++it) + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( { - // escape the key name to be used in a JSON patch - const auto key = json_pointer::escape(it.key()); - - if (target.find(it.key()) != target.end()) - { - // recursive call to compare object values at key it - auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); - result.insert(result.end(), temp_diff.begin(), temp_diff.end()); - } - else - { - // found a key that is not in o -> remove it - result.push_back(object( - { - {"op", "remove"}, {"path", path + "/" + key} - })); - } - } - - // second pass: traverse other object's elements - for (auto it = target.cbegin(); it != target.cend(); ++it) - { - if (source.find(it.key()) == source.end()) - { - // found a key that is not in this -> add it - const auto key = json_pointer::escape(it.key()); - result.push_back( - { - {"op", "add"}, {"path", path + "/" + key}, - {"value", it.value()} - }); - } - } - - break; + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; } - default: + // add other remaining elements + while (i < target.size()) { - // both primitive type: replace value result.push_back( { - {"op", "replace"}, {"path", path}, {"value", target} + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} }); - break; + ++i; } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; } } @@ -20397,10 +20832,11 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #undef JSON_LIKELY #undef JSON_UNLIKELY #undef JSON_DEPRECATED +#undef JSON_NODISCARD #undef JSON_HAS_CPP_14 #undef JSON_HAS_CPP_17 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION #undef NLOHMANN_BASIC_JSON_TPL -#endif +#endif // INCLUDE_NLOHMANN_JSON_HPP_ From fc46f8fc5ed34f789a73ef5fedf609bfb882ce53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 14:55:12 +0200 Subject: [PATCH 0152/2548] flake-registry: Use nixpkgs 19.03 --- flake-registry.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake-registry.json b/flake-registry.json index 422f77675..ae94f1df2 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -7,7 +7,7 @@ "uri": "github:tweag/nix/flakes" }, "nixpkgs": { - "uri": "github:edolstra/nixpkgs/flake" + "uri": "github:edolstra/nixpkgs/release-19.03" } }, "version": 1 From 7ec7bad2f8174c67ced4a0a3e88f61c1f88affd0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 15:52:40 +0200 Subject: [PATCH 0153/2548] Update flake.lock --- flake.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.lock b/flake.lock index 3154006c8..9848cb996 100644 --- a/flake.lock +++ b/flake.lock @@ -2,7 +2,7 @@ "nonFlakeRequires": {}, "requires": { "nixpkgs": { - "uri": "github:edolstra/nixpkgs/f10e8a02eb7fa2b4a070f30cf87f4efcc7f3186d" + "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" } }, "version": 1 From cc51e37ad09a1dac0a58c543ffe275ddc18819aa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Apr 2019 16:07:37 +0200 Subject: [PATCH 0154/2548] Use "." as the default installable This makes e.g. "nix build" do something more or less reasonable (namely, build the default package of the flake in the current directory). --- src/nix/build.cc | 1 - src/nix/command.hh | 8 ++++++-- src/nix/installables.cc | 4 ++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index d6a6a8071..07b6b2f02 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -3,7 +3,6 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "primops/flake.hh" using namespace nix; diff --git a/src/nix/command.hh b/src/nix/command.hh index a52fbb9ba..6d43261ac 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -108,6 +108,8 @@ struct InstallablesCommand : virtual Args, SourceExprCommand void prepare() override; + virtual bool useDefaultInstallables() { return true; } + private: std::vector _installables; @@ -119,14 +121,14 @@ struct InstallableCommand : virtual Args, SourceExprCommand InstallableCommand() { - expectArg("installable", &_installable); + expectArg("installable", &_installable, true); } void prepare() override; private: - std::string _installable; + std::string _installable{"."}; }; /* A command that operates on zero or more store paths. */ @@ -146,6 +148,8 @@ public: virtual void run(ref store, Paths storePaths) = 0; void run(ref store) override; + + bool useDefaultInstallables() override { return !all; } }; /* A command that operates on exactly one store path. */ diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 9d87c70c3..c3ca87aa7 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -334,6 +334,10 @@ PathSet toDerivations(ref store, void InstallablesCommand::prepare() { + if (_installables.empty() && !file && useDefaultInstallables()) + // FIXME: commands like "nix install" should not have a + // default, probably. + _installables.push_back("."); installables = parseInstallables(getStore(), _installables); } From e9c42c06ef9a1f7af7c34162cc91a0dbeb746202 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 25 Apr 2019 10:49:56 +0200 Subject: [PATCH 0155/2548] Fixed lookupFlake bug --- src/libexpr/primops/flake.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index d898da1e9..f6744a1f7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -146,15 +146,14 @@ std::shared_ptr getFlagRegistry() return std::make_shared(); } -// This always returns a vector with globalReg, userReg, localReg, flakeReg. +// This always returns a vector with flakeRef, userReg, globalReg. // If one of them doesn't exist, the registry is left empty but does exist. const Registries EvalState::getFlakeRegistries() { Registries registries; - registries.push_back(getGlobalRegistry()); // TODO (Nick): Doesn't this break immutability? - registries.push_back(getUserRegistry()); - registries.push_back(std::make_shared()); // local registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalRegistry()); return registries; } @@ -169,8 +168,10 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const if (i != registry->entries.end()) { auto newRef = i->second; if (std::get_if(&flakeRef.data)) { - if (flakeRef.ref) newRef.ref = flakeRef.ref; - if (flakeRef.rev) newRef.rev = flakeRef.rev; + if (flakeRef.ref || flakeRef.rev) { + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + } } std::string errorMsg = "found cycle in flake registries: "; for (FlakeRef oldRef : pastSearches) { From d867e1804a39174f21d50095d9e6bc4a87190e16 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 17 Apr 2019 14:03:04 +0200 Subject: [PATCH 0156/2548] Fix printing FlakeList --- src/nix/flake.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6459df019..a25493f79 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -30,9 +30,14 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs stopProgressBar(); - for (auto & registry : registries) - for (auto & entry : registry->entries) - std::cout << entry.first << " " << entry.second << "\n"; + for (auto & entry : registries[0]->entries) + std::cout << entry.first.to_string() << " flags " << entry.second.to_string() << "\n"; + + for (auto & entry : registries[1]->entries) + std::cout << entry.first.to_string() << " user " << entry.second.to_string() << "\n"; + + for (auto & entry : registries[2]->entries) + std::cout << entry.first.to_string() << " global " << entry.second.to_string() << "\n"; } }; @@ -146,7 +151,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); - nix::Flake flake = nix::getFlake(*evalState, FlakeRef(flakeUri), true); + Flake flake = getFlake(*evalState, FlakeRef(flakeUri), true); printFlakeInfo(flake, json); } }; From 35d1c95f7f6fc744a829bb25b75c15cc2da31e99 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Tue, 30 Apr 2019 11:03:31 +0200 Subject: [PATCH 0157/2548] Fix flag registry order --- src/libexpr/primops/flake.cc | 2 +- src/libexpr/primops/flake.hh | 4 ++++ src/nix/flake.cc | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f6744a1f7..84e878f75 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -146,7 +146,7 @@ std::shared_ptr getFlagRegistry() return std::make_shared(); } -// This always returns a vector with flakeRef, userReg, globalReg. +// This always returns a vector with flakeReg, userReg, globalReg. // If one of them doesn't exist, the registry is left empty but does exist. const Registries EvalState::getFlakeRegistries() { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 4e0d3b646..8e9af5843 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -5,6 +5,10 @@ namespace nix { +static const size_t FLAG_REGISTRY = 0; +static const size_t USER_REGISTRY = 1; +static const size_t GLOBAL_REGISTRY = 2; + struct Value; class EvalState; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a25493f79..a80338fd3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -30,13 +30,13 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs stopProgressBar(); - for (auto & entry : registries[0]->entries) + for (auto & entry : registries[FLAG_REGISTRY]->entries) std::cout << entry.first.to_string() << " flags " << entry.second.to_string() << "\n"; - for (auto & entry : registries[1]->entries) + for (auto & entry : registries[USER_REGISTRY]->entries) std::cout << entry.first.to_string() << " user " << entry.second.to_string() << "\n"; - for (auto & entry : registries[2]->entries) + for (auto & entry : registries[GLOBAL_REGISTRY]->entries) std::cout << entry.first.to_string() << " global " << entry.second.to_string() << "\n"; } }; From 24b35bf9e7219feba9399466c41801f0ace3ef74 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 21 Mar 2019 09:30:16 +0100 Subject: [PATCH 0158/2548] Fixed issue #13 --- src/libexpr/common-eval-args.cc | 9 +++++++++ src/libexpr/common-eval-args.hh | 2 ++ src/libexpr/eval.hh | 7 +++++++ src/libexpr/primops/flake.cc | 22 +++++++++++----------- src/nix/build.cc | 2 +- src/nix/flake.cc | 1 + 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e5400052..8e94d358e 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -33,6 +33,15 @@ MixEvalArgs::MixEvalArgs() .handler([&](std::vector ss) { evalSettings.pureEval = false; }); + + mkFlag() + .longName("override-flake") + .labels({"original-ref", "resolved-ref"}) + .description("override a flake registry value") + .arity(2) + .handler([&](std::vector ss) { + registryOverrides.push_back(std::make_pair(ss[0], ss[1])); + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index be7fda783..54fb731de 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -16,6 +16,8 @@ struct MixEvalArgs : virtual Args Strings searchPath; + std::vector> registryOverrides; + private: std::map autoArgs; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3f537da89..44988cd70 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -63,6 +63,8 @@ typedef std::list SearchPath; /* Initialise the Boehm GC, if applicable. */ void initGC(); +typedef std::vector> RegistryOverrides; + class EvalState { @@ -89,6 +91,9 @@ public: const ref store; + RegistryOverrides registryOverrides; + + private: SrcToStore srcToStore; @@ -211,6 +216,8 @@ public: path. Nothing is copied to the store. */ Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + void addRegistryOverrides(RegistryOverrides overrides) { registryOverrides = overrides; } + public: /* The base environment, containing the builtin functions and diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f6744a1f7..5af0c49c7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -140,10 +140,13 @@ std::shared_ptr getUserRegistry() return readRegistry(getUserRegistryPath()); } -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) { - // TODO (Nick): Implement this. - return std::make_shared(); + auto flagRegistry = std::make_shared(); + for (auto const & x : registryOverrides) { + flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); + } + return flagRegistry; } // This always returns a vector with flakeRef, userReg, globalReg. @@ -151,7 +154,7 @@ std::shared_ptr getFlagRegistry() const Registries EvalState::getFlakeRegistries() { Registries registries; - registries.push_back(getFlagRegistry()); + registries.push_back(getFlagRegistry(registryOverrides)); registries.push_back(getUserRegistry()); registries.push_back(getGlobalRegistry()); return registries; @@ -357,9 +360,8 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, RegistryAccess registryAccess, bool isTopFlake) { - Flake flake = getFlake(state, topRef, - registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake)); - + bool allowRegistries = registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake); + Flake flake = getFlake(state, topRef, allowRegistries); LockFile lockFile; if (isTopFlake) @@ -405,10 +407,8 @@ static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) void updateLockFile(EvalState & state, const Path & path) { - // FIXME: don't copy 'path' to the store (especially since we - // dirty it immediately afterwards). - - FlakeRef flakeRef = FlakeRef(path); // FIXME: ugly + // FIXME: We are writing the lockfile to the store here! Very bad practice! + FlakeRef flakeRef = FlakeRef(path); auto lockFile = makeLockFile(state, flakeRef); writeLockFile(lockFile, path + "/flake.lock"); diff --git a/src/nix/build.cc b/src/nix/build.cc index 07b6b2f02..c08ec0e62 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -54,7 +54,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand auto buildables = build(store, dryRun ? DryRun : Build, installables); auto evalState = std::make_shared(searchPath, store); - + evalState->addRegistryOverrides(registryOverrides); if (dryRun) return; for (size_t i = 0; i < buildables.size(); ++i) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a25493f79..00c1c384c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -92,6 +92,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs void run(nix::ref store) override { auto evalState = std::make_shared(searchPath, store); + evalState->addRegistryOverrides(registryOverrides); FlakeRef flakeRef(flakeUri); From cbc7d9a4124343c7ba68b695b5e9b3c8188fb267 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Tue, 30 Apr 2019 22:43:24 -0400 Subject: [PATCH 0159/2548] findRootsNoTemp: fixes comment about findRuntimeRoots The NIX_ROOT_FINDER environment variable was removed in 3c46fe62b833a4e66845665edc99555022d3d98c when porting from perl to C. --- src/libstore/gc.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d8a5da0d4..26e2b0dca 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -326,10 +326,9 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor) findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ findRuntimeRoots(roots, censor); } From eba85e23670c57448aaf29d2fedec356e4cd2fe7 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 1 May 2019 10:34:23 +0200 Subject: [PATCH 0160/2548] WIP: still need to adapt flakeref parsing --- src/libexpr/primops/flake.cc | 22 ++++++++++++---------- src/libexpr/primops/flakeref.cc | 1 + src/libexpr/primops/flakeref.hh | 11 ++++++----- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 88a0293e3..e806ef6c7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -194,11 +194,8 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const return flakeRef; } -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bool impureIsAllowed = false) +static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool impureIsAllowed = false) { - FlakeRef fRef = lookupFlake(state, flakeRef, - impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); - if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) throw Error("requested to fetch mutable flake '%s' in pure mode", fRef); @@ -266,26 +263,31 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef flakeRef, bo // This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + + FlakeSourceInfo sourceInfo = fetchFlake(state, resolvedRef, impureIsAllowed); debug("got flake source '%s' with revision %s", sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + resolvedRef = sourceInfo.flakeRef; // `resolvedRef` is now immutable + state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(sourceInfo.storePath); - Flake flake(flakeRef, std::move(sourceInfo)); - if (std::get_if(&flakeRef.data)) { + Flake flake(resolvedRef, std::move(sourceInfo)); + if (std::get_if(&resolvedRef.data)) { // FIXME: ehm? if (flake.sourceInfo.rev) - flake.ref = FlakeRef(flakeRef.baseRef().to_string() + flake.ref = FlakeRef(resolvedRef.baseRef().to_string() + "/" + flake.sourceInfo.rev->to_string(Base16, false)); } - Path flakeFile = sourceInfo.storePath + "/flake.nix"; + Path flakeFile = sourceInfo.storePath + resolvedRef.subdir + "/flake.nix"; if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", flakeRef); + throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", resolvedRef); Value vInfo; state.evalFile(flakeFile, vInfo); // FIXME: symlink attack diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index b91bbee2a..022535515 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -139,6 +139,7 @@ std::string FlakeRef::to_string() const string += (ref ? "/" + *ref : "") + (rev ? "/" + rev->to_string(Base16, false) : ""); + if (subdir != "") string += "?dir=" + subdir; return string; } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 51fdc3b70..299094634 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -69,7 +69,7 @@ namespace nix { https://example.org/my/repo.git https://example.org/my/repo.git?ref=release-1.2.3 https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - git://github.com/edolstra/dwarffs.git\?ref=flake\&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 + git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 * /path.git(\?attr(&attr)*)? @@ -144,17 +144,18 @@ struct FlakeRef std::optional ref; std::optional rev; + Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. bool operator<(const FlakeRef & flakeRef) const { - return std::make_tuple(this->data, ref, rev) < - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + return std::make_tuple(data, ref, rev, subdir) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); } bool operator==(const FlakeRef & flakeRef) const { - return std::make_tuple(this->data, ref, rev) == - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev); + return std::make_tuple(data, ref, rev, subdir) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); } // Parse a flake URI. From 9b3069a88ceff3646d254fa403f41be889e6423d Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Tue, 30 Apr 2019 12:47:15 +0200 Subject: [PATCH 0161/2548] Fuzzymatching Fixed issue #61 --- src/libexpr/primops/flake.cc | 42 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 88a0293e3..cc93eee36 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -161,7 +161,23 @@ const Registries EvalState::getFlakeRegistries() } static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, - std::vector pastSearches = {}) + std::vector pastSearches = {}); + +FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector pastSearches) +{ + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector pastSearches) { if (registries.empty() && !flakeRef.isDirect()) throw Error("indirect flake reference '%s' is not allowed", flakeRef); @@ -170,21 +186,15 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const auto i = registry->entries.find(flakeRef); if (i != registry->entries.end()) { auto newRef = i->second; - if (std::get_if(&flakeRef.data)) { - if (flakeRef.ref || flakeRef.rev) { - newRef.ref = flakeRef.ref; - newRef.rev = flakeRef.rev; - } - } - std::string errorMsg = "found cycle in flake registries: "; - for (FlakeRef oldRef : pastSearches) { - errorMsg += oldRef.to_string(); - if (oldRef == newRef) - throw Error(errorMsg); - errorMsg += " - "; - } - pastSearches.push_back(newRef); - return lookupFlake(state, newRef, registries, pastSearches); + return updateFlakeRef(state, newRef, registries, pastSearches); + } + + auto j = registry->entries.find(flakeRef.baseRef()); + if (j != registry->entries.end()) { + auto newRef = j->second; + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + return updateFlakeRef(state, newRef, registries, pastSearches); } } From 00db8d4549b1e87db45395e157dec3f729bdc071 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 May 2019 14:24:22 +0200 Subject: [PATCH 0162/2548] Support 'dir' parameters in https and ssh flake URIs --- src/libexpr/primops/flakeref.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 022535515..66eab4db6 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -30,7 +30,9 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; -const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*"; +// FIXME: support escaping in query string. +// Note: '/' is not a valid query parameter, but so what... +const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*"; FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) { @@ -97,6 +99,9 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) if (!std::regex_match(value, refRegex2)) throw Error("invalid Git ref '%s'", value); ref = value; + } else if (name == "dir") { + // FIXME: validate value; should not contain relative paths + subdir = value; } else // FIXME: should probably pass through unknown parameters throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); From 43408d3cd6e254c1c69eb9cf9de4de042b986ab6 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 1 May 2019 16:24:33 +0200 Subject: [PATCH 0163/2548] flake.lock now uses flakeRef.subdir --- src/libexpr/primops/flake.cc | 21 ++++++++++++--------- src/libexpr/primops/flake.hh | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e806ef6c7..41b3f6d28 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -285,7 +285,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe + "/" + flake.sourceInfo.rev->to_string(Base16, false)); } - Path flakeFile = sourceInfo.storePath + resolvedRef.subdir + "/flake.nix"; + Path flakeFile = sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"; if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", resolvedRef); @@ -367,7 +367,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile lockFile; if (isTopFlake) - lockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack + lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack ResolvedFlake deps(flake); @@ -407,16 +407,19 @@ static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) return lockFile; } -void updateLockFile(EvalState & state, const Path & path) +void updateLockFile(EvalState & state, const FlakeUri & flakeUri) { // FIXME: We are writing the lockfile to the store here! Very bad practice! - FlakeRef flakeRef = FlakeRef(path); - auto lockFile = makeLockFile(state, flakeRef); - writeLockFile(lockFile, path + "/flake.lock"); + FlakeRef flakeRef = FlakeRef(flakeUri); + if (auto refData = std::get_if(flakeRef)) { + auto lockFile = makeLockFile(state, flakeRef); + writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock"); - // Hack: Make sure that flake.lock is visible to Git. Otherwise, - // exportGit will fail to copy it to the Nix store. - runProgram("git", true, { "-C", path, "add", "flake.lock" }); + // Hack: Make sure that flake.lock is visible to Git. Otherwise, + // exportGit will fail to copy it to the Nix store. + runProgram("git", true, { "-C", refData->path, "add", flakeRef.subDir + "/flake.lock" }); + } else + throw Error("flakeUri %s can't be updated because it is not a path", flakeUri); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 8e9af5843..6329c36ec 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -90,7 +90,7 @@ struct ResolvedFlake ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); -void updateLockFile(EvalState &, const Path & path); +void updateLockFile(EvalState &, const FlakeUri &); void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path); } From ab9e47284a46f1a933d84c9c66f8fb4ba6c4ba34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 May 2019 18:07:36 +0200 Subject: [PATCH 0164/2548] Improve error message --- src/libexpr/primops/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 41b3f6d28..293e5ad0b 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -287,7 +287,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe Path flakeFile = sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"; if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a 'flake.nix' file", resolvedRef); + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); Value vInfo; state.evalFile(flakeFile, vInfo); // FIXME: symlink attack From fa88f7152070d4c886b512de00691da709bc7229 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 May 2019 20:23:39 +0200 Subject: [PATCH 0165/2548] Validate 'dir=' parameters We reject any path element starting with a '.' (mostly to reject '.' and '..'). --- src/libexpr/primops/flakeref.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 66eab4db6..56ba58d09 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -34,6 +34,11 @@ const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex // Note: '/' is not a valid query parameter, but so what... const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*"; +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; + FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) { // FIXME: could combine this into one regex. @@ -55,6 +60,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) static std::regex refRegex2(refRegex, std::regex::ECMAScript); + static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); + std::cmatch match; if (std::regex_match(uri.c_str(), match, flakeRegex)) { IsAlias d; @@ -100,7 +107,8 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) throw Error("invalid Git ref '%s'", value); ref = value; } else if (name == "dir") { - // FIXME: validate value; should not contain relative paths + if (!std::regex_match(value, subDirRegex2)) + throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); subdir = value; } else // FIXME: should probably pass through unknown parameters @@ -124,6 +132,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { std::string string; + if (auto refData = std::get_if(&data)) string = refData->alias; @@ -142,9 +151,12 @@ std::string FlakeRef::to_string() const else abort(); + // FIXME: need to use ?rev etc. for IsGit URIs. string += (ref ? "/" + *ref : "") + (rev ? "/" + rev->to_string(Base16, false) : ""); + if (subdir != "") string += "?dir=" + subdir; + return string; } From a37436d7929f37fb390837419d166a81559abb3e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 May 2019 20:38:41 +0200 Subject: [PATCH 0166/2548] Accept empty directories --- src/libexpr/primops/flake.cc | 5 ++++- src/libexpr/primops/flakeref.cc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 293e5ad0b..de692f167 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -285,7 +285,10 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe + "/" + flake.sourceInfo.rev->to_string(Base16, false)); } - Path flakeFile = sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"; + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.storePath)) + throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 56ba58d09..b7a20a170 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -107,7 +107,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) throw Error("invalid Git ref '%s'", value); ref = value; } else if (name == "dir") { - if (!std::regex_match(value, subDirRegex2)) + if (value != "" && !std::regex_match(value, subDirRegex2)) throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); subdir = value; } else From 5d6e8c008ba8716f16ddfad954663d9e732f8556 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 May 2019 20:43:16 +0200 Subject: [PATCH 0167/2548] Allow 'dir' parameter in github: URIs E.g. 'github:edolstra/dwarffs/flake?dir=foo/bar'. --- src/libexpr/primops/flakeref.cc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index b7a20a170..141d61c0d 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -33,12 +33,14 @@ const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex // FIXME: support escaping in query string. // Note: '/' is not a valid query parameter, but so what... const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*"; +const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))"; // 'dir' path elements cannot start with a '.'. We also reject // potentially dangerous characters like ';'. const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; + FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) { // FIXME: could combine this into one regex. @@ -48,14 +50,15 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::regex::ECMAScript); static std::regex githubRegex( - "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?" + + paramsRegex + "?", std::regex::ECMAScript); static std::regex uriRegex( "((" + schemeRegex + "):" + "(?://(" + authorityRegex + "))?" + "(" + pathRegex + "))" + - "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?", + paramsRegex + "?", std::regex::ECMAScript); static std::regex refRegex2(refRegex, std::regex::ECMAScript); @@ -85,6 +88,18 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) else if (match[4].matched) { ref = match[4]; } + for (auto & param : tokenizeString(match[5], "&")) { + auto n = param.find('='); + assert(n != param.npos); + std::string name(param, 0, n); + std::string value(param, n + 1); + if (name == "dir") { + if (value != "" && !std::regex_match(value, subDirRegex2)) + throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); + subdir = value; + } else + throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); + } data = d; } From e0d4aa75fc8957347215f3c52366f47f1e3d6d6e Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Thu, 2 May 2019 08:40:00 +0200 Subject: [PATCH 0168/2548] Fixed compile errors --- src/libexpr/primops/flake.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index de692f167..0d9ef36ba 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -370,7 +370,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile lockFile; if (isTopFlake) - lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack + lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.ref.subdir + "/flake.lock"); // FIXME: symlink attack ResolvedFlake deps(flake); @@ -414,13 +414,13 @@ void updateLockFile(EvalState & state, const FlakeUri & flakeUri) { // FIXME: We are writing the lockfile to the store here! Very bad practice! FlakeRef flakeRef = FlakeRef(flakeUri); - if (auto refData = std::get_if(flakeRef)) { + if (auto refData = std::get_if(&flakeRef.data)) { auto lockFile = makeLockFile(state, flakeRef); writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git. Otherwise, // exportGit will fail to copy it to the Nix store. - runProgram("git", true, { "-C", refData->path, "add", flakeRef.subDir + "/flake.lock" }); + runProgram("git", true, { "-C", refData->path, "add", flakeRef.subdir + "/flake.lock" }); } else throw Error("flakeUri %s can't be updated because it is not a path", flakeUri); } From 2b8c63f303de7d1da133cb3e9a00d509dca40f57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 20:22:14 +0200 Subject: [PATCH 0169/2548] Add 'nix dev-shell' and 'nix print-dev-env' command 'nix dev-shell' is intended to replace nix-shell. It supports flakes, e.g. $ nix dev-shell nixpkgs:hello starts a bash shell providing an environment for building 'hello'. Like Lorri (and unlike nix-shell), it computes the build environment by building a modified top-level derivation that writes the environment after running $stdenv/setup to $out and exits. This provides some caching, so it's faster than nix-shell in some cases (especially for packages with lots of dependencies, where the setup script takes a long time). There also is a command 'nix print-dev-env' that prints out shell code for setting up the build environment in an existing shell, e.g. $ . <(nix print-dev-env nixpkgs:hello) https://github.com/tweag/nix/issues/21 --- src/nix/shell.cc | 276 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 src/nix/shell.cc diff --git a/src/nix/shell.cc b/src/nix/shell.cc new file mode 100644 index 000000000..14d88faeb --- /dev/null +++ b/src/nix/shell.cc @@ -0,0 +1,276 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct BuildEnvironment +{ + // FIXME: figure out which vars should be exported. + std::map env; + std::map functions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + auto lines = tokenizeString(readFile(path), "\n"); + + auto getLine = + [&]() { + if (lines.empty()) + throw Error("shell environment '%s' ends unexpectedly", path); + auto line = lines.front(); + lines.pop_front(); + return line; + }; + + while (!lines.empty()) { + auto line = getLine(); + + auto eq = line.find('='); + if (eq != std::string::npos) { + std::string name(line, 0, eq); + std::string value(line, eq + 1); + // FIXME: parse arrays + res.env.insert({name, value}); + } + + else if (hasSuffix(line, " () ")) { + std::string name(line, 0, line.size() - 4); + // FIXME: validate name + auto l = getLine(); + if (l != "{ ") throw Error("shell environment '%s' has unexpected line '%s'", path, l); + std::string body; + while ((l = getLine()) != "}") { + body += l; + body += '\n'; + } + res.functions.insert({name, body}); + } + + else throw Error("shell environment '%s' has unexpected line '%s'", path, line); + } + + return res; +} + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +BuildEnvironment getDerivationEnvironment(ref store, Derivation drv) +{ + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); + + drv.args = {"-c", "set -e; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + // FIXME: handle structured attrs + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = drv.env["name"] + "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + drv.outputs["out"] = DerivationOutput("", "", ""); + Hash h = hashDerivationModulo(*store, drv); + Path shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs["out"].path = shellOutPath; + drv.env["out"] = shellOutPath; + Path shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({shellDrvPath2}); + + assert(store->isValidPath(shellOutPath)); + + return readEnvironment(shellOutPath); +} + +struct Common : InstallableCommand +{ + /* + std::set keepVars{ + "DISPLAY", + "HOME", + "IN_NIX_SHELL", + "LOGNAME", + "NIX_BUILD_SHELL", + "PAGER", + "PATH", + "TERM", + "TZ", + "USER", + }; + */ + + std::set ignoreVars{ + "BASHOPTS", + "EUID", + "NIX_BUILD_TOP", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "TEMP", + "TEMPDIR", + "TMP", + "TMPDIR", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + for (auto & i : buildEnvironment.env) { + // FIXME: shellEscape + // FIXME: figure out what to export + // FIXME: handle arrays + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) + out << fmt("export %s=%s\n", i.first, i.second); + } + + for (auto & i : buildEnvironment.functions) { + out << fmt("%s () {\n%s\n}\n", i.first, i.second); + } + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + } +}; + +std::pair createTempFile(const Path & prefix = "nix") +{ + Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + +struct CmdDevShell : Common +{ + + std::string name() override + { + return "dev-shell"; + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix dev-shell nixpkgs:hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix dev-shell" + }, + }; + } + + void run(ref store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL", "bash"); + + auto args = Strings{baseNameOf(shell), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + + std::string name() override + { + return "print-dev-env"; + } + + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs:hello)" + }, + }; + } + + void run(ref store) override + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); From 7dcf5b011a0942ecf953f2b607c4c8d0e9e652c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:09:52 +0200 Subject: [PATCH 0170/2548] Add function for quoting strings --- src/libstore/build.cc | 2 +- src/libstore/store-api.cc | 7 +------ src/libutil/util.cc | 22 ---------------------- src/libutil/util.hh | 22 ++++++++++++++++++++-- 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 53a0c743b..9730c75e2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1805,7 +1805,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..8fabeeea4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -726,12 +726,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f4f86c5c8..5598415f5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1167,28 +1167,6 @@ template StringSet tokenizeString(const string & s, const string & separators); template vector tokenizeString(const string & s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..8bd57d2e4 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -334,8 +334,26 @@ template C tokenizeString(const string & s, const string & separators = /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ From 2919c496ea10a99d08baffbef485cfb719233b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:10:13 +0200 Subject: [PATCH 0171/2548] nix dev-shell: Use 'provides.devShell' by default Thus $ nix dev-shell will now build the 'provides.devShell' attribute from the flake in the current directory. If it doesn't exist, it falls back to 'provides.defaultPackage'. --- flake.nix | 5 +++++ shell.nix | 6 ++++-- src/nix/command.hh | 5 +++++ src/nix/installables.cc | 42 ++++++++++++++++++++++++++++++----------- src/nix/shell.cc | 6 ++++++ 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 95ec5d952..ab316c7c6 100644 --- a/flake.nix +++ b/flake.nix @@ -17,5 +17,10 @@ packages.nix = hydraJobs.build.x86_64-linux; defaultPackage = packages.nix; + + devShell = import ./shell.nix { + nixpkgs = deps.nixpkgs; + }; + }; } diff --git a/shell.nix b/shell.nix index 8167f87a2..d7e63bad3 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,8 @@ -{ useClang ? false }: +{ useClang ? false +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz +}: -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; +with import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/nix/command.hh b/src/nix/command.hh index 6d43261ac..640c6cd16 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -88,6 +88,11 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs std::shared_ptr parseInstallable( ref store, const std::string & installable); + virtual Strings getDefaultFlakeAttrPaths() + { + return {"defaultPackage"}; + } + private: std::shared_ptr evalState; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c3ca87aa7..db67952e1 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -142,13 +142,18 @@ struct InstallableAttrPath : InstallableValue struct InstallableFlake : InstallableValue { FlakeRef flakeRef; - std::string attrPath; + Strings attrPaths; + bool searchPackages = false; - InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, const std::string & attrPath) - : InstallableValue(cmd), flakeRef(flakeRef), attrPath(attrPath) + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) { } - std::string what() override { return flakeRef.to_string() + ":" + attrPath; } + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, std::string attrPath) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, searchPackages(true) + { } + + std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } Value * toValue(EvalState & state) override { @@ -166,18 +171,31 @@ struct InstallableFlake : InstallableValue auto emptyArgs = state.allocBindings(0); - if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + // As a convenience, look for the attribute in + // 'provides.packages'. + if (searchPackages) { + if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + try { + auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + } + + // Otherwise, look for it in 'provides'. + for (auto & attrPath : attrPaths) { try { - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *aPackages->value); + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); state.forceValue(*v); return v; } catch (AttrPathNotFound & e) { } } - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); - state.forceValue(*v); - return v; + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } }; @@ -216,7 +234,8 @@ std::vector> SourceExprCommand::parseInstallables( else if (hasPrefix(s, "nixpkgs.")) { bool static warned; warnOnce(warned, "the syntax 'nixpkgs.' is deprecated; use 'nixpkgs:' instead"); - result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), std::string(s, 8))); + result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), + Strings{"packages." + std::string(s, 8)})); } else if ((colon = s.rfind(':')) != std::string::npos) { @@ -233,7 +252,8 @@ std::vector> SourceExprCommand::parseInstallables( if (storePath != "") result.push_back(std::make_shared(storePath)); else - result.push_back(std::make_shared(*this, FlakeRef(s, true), "defaultPackage")); + result.push_back(std::make_shared(*this, FlakeRef(s, true), + getDefaultFlakeAttrPaths())); } else diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 14d88faeb..2e17111d6 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -125,6 +125,7 @@ struct Common : InstallableCommand "BASHOPTS", "EUID", "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", "PPID", "PWD", "SHELLOPTS", @@ -156,6 +157,11 @@ struct Common : InstallableCommand for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); } + + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell", "defaultPackage"}; + } }; std::pair createTempFile(const Path & prefix = "nix") From dea18ff99913349ef0f54b80d45c8dfdc8b31f65 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:13:19 +0200 Subject: [PATCH 0172/2548] nix dev-shell: Execute shellHook --- src/nix/shell.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 2e17111d6..d3ecf8de4 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -156,6 +156,8 @@ struct Common : InstallableCommand out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; } Strings getDefaultFlakeAttrPaths() override From 8ec77614f63e14d1869734b0d21a646667bbf88b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:28:41 +0200 Subject: [PATCH 0173/2548] Move createTempFile to libutil --- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 12 ++++++++---- src/nix/shell.cc | 10 ---------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5598415f5..75b73fcfa 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -461,6 +461,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + static Lazy getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8bd57d2e4..6c9d7c2eb 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -118,10 +118,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -199,6 +195,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair createTempFile(const Path & prefix = "nix"); + + class Pipe { public: diff --git a/src/nix/shell.cc b/src/nix/shell.cc index d3ecf8de4..95028f10e 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -166,16 +166,6 @@ struct Common : InstallableCommand } }; -std::pair createTempFile(const Path & prefix = "nix") -{ - Path tmpl(getEnv("TMPDIR", "/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - return {std::move(fd), tmpl}; -} - struct CmdDevShell : Common { From 7ba0f98e644c31ce0c16db10aa87f896937e0ddf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 May 2019 21:28:52 +0200 Subject: [PATCH 0174/2548] nix dev-shell: Less purity --- src/nix/shell.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 95028f10e..0813d122c 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -124,6 +124,7 @@ struct Common : InstallableCommand std::set ignoreVars{ "BASHOPTS", "EUID", + "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "PPID", @@ -134,11 +135,15 @@ struct Common : InstallableCommand "TEMPDIR", "TMP", "TMPDIR", + "TZ", "UID", }; void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { + out << "export IN_NIX_SHELL=1\n"; + out << "nix_saved_PATH=\"$PATH\"\n"; + for (auto & i : buildEnvironment.env) { // FIXME: shellEscape // FIXME: figure out what to export @@ -147,6 +152,8 @@ struct Common : InstallableCommand out << fmt("export %s=%s\n", i.first, i.second); } + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + for (auto & i : buildEnvironment.functions) { out << fmt("%s () {\n%s\n}\n", i.first, i.second); } From f9a2ea44867cd1dbb408bca4df0ced806137b7f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 May 2019 10:44:32 +0200 Subject: [PATCH 0175/2548] Fix "Bad system call" running i686-linux binaries on x86_64-linux To determine which seccomp filters to install, we were incorrectly using settings.thisSystem, which doesn't denote the actual system when --system is used. Fixes #2791. --- src/libstore/build.cc | 8 ++++---- src/libutil/util.cc | 3 +++ src/libutil/util.hh | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dbadfacd5..91eb97dfb 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2510,17 +2510,17 @@ void setupSeccomp() seccomp_release(ctx); }); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) throw SysError("unable to add X32 seccomp architecture"); - if (settings.thisSystem == "aarch64-linux" && + if (nativeSystem == "aarch64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes."); + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e3dcd246c..a71705665 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -38,6 +38,9 @@ extern char * * environ; namespace nix { +const std::string nativeSystem = SYSTEM; + + BaseError & BaseError::addPrefix(const FormatOrString & fs) { prefix_ = fs.s + prefix_; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..54936a5cb 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -30,6 +30,10 @@ struct Sink; struct Source; +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); From f8c4742c2f92c7ba4dbe0911fb91ddce33c6f98a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 May 2019 13:15:13 +0200 Subject: [PATCH 0176/2548] Fix 'git add' when subdir is empty --- src/libexpr/primops/flake.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1c20d33bb..ab797fe01 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -430,7 +430,9 @@ void updateLockFile(EvalState & state, const FlakeUri & flakeUri) // Hack: Make sure that flake.lock is visible to Git. Otherwise, // exportGit will fail to copy it to the Nix store. - runProgram("git", true, { "-C", refData->path, "add", flakeRef.subdir + "/flake.lock" }); + runProgram("git", true, + { "-C", refData->path, "add", + (flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" }); } else throw Error("flakeUri %s can't be updated because it is not a path", flakeUri); } From 3f192ac80ca421888c668896b63485486e1397ae Mon Sep 17 00:00:00 2001 From: Daniel Schaefer Date: Fri, 3 May 2019 14:30:29 +0200 Subject: [PATCH 0177/2548] Add builtins.hashFile For text files it is possible to do it like so: `builtins.hashString "sha256" (builtins.readFile /tmp/a)` but that doesn't work for binary files. With builtins.hashFile any kind of file can be conveniently hashed. --- doc/manual/expressions/builtins.xml | 13 +++++++++++++ src/libexpr/primops.cc | 15 +++++++++++++++ tests/lang/binary-data | Bin 0 -> 1024 bytes tests/lang/eval-fail-hashfile-missing.nix | 5 +++++ tests/lang/eval-okay-hash.exp | 1 - tests/lang/eval-okay-hashfile.exp | 1 + tests/lang/eval-okay-hashfile.nix | 4 ++++ tests/lang/eval-okay-hashstring.exp | 1 + ...al-okay-hash.nix => eval-okay-hashstring.nix} | 0 9 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/lang/binary-data create mode 100644 tests/lang/eval-fail-hashfile-missing.nix create mode 100644 tests/lang/eval-okay-hashfile.exp create mode 100644 tests/lang/eval-okay-hashfile.nix create mode 100644 tests/lang/eval-okay-hashstring.exp rename tests/lang/{eval-okay-hash.nix => eval-okay-hashstring.nix} (100%) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 0fb5261b3..a87639a07 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -705,6 +705,19 @@ builtins.genList (x: x * x) 5 + + builtins.hashFile + type p + + Return a base-16 representation of the + cryptographic hash of the file at path p. The + hash algorithm specified by type must + be one of "md5", "sha1", + "sha256" or "sha512". + + + + builtins.head list diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 39073725e..06f577f36 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -923,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2202,6 +2216,7 @@ void EvalState::createBaseEnv() addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); diff --git a/tests/lang/binary-data b/tests/lang/binary-data new file mode 100644 index 0000000000000000000000000000000000000000..06d7405020018ddf3cacee90fd4af10487da3d20 GIT binary patch literal 1024 ScmZQz7zLvtFd70QH3R?z00031 literal 0 HcmV?d00001 diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix new file mode 100644 index 000000000..42fb1ec7e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -0,0 +1,5 @@ +let + paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) + diff --git a/tests/lang/eval-okay-hash.exp b/tests/lang/eval-okay-hash.exp index d720a082d..e69de29bb 100644 --- a/tests/lang/eval-okay-hash.exp +++ b/tests/lang/eval-okay-hash.exp @@ -1 +0,0 @@ -[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/lang/eval-okay-hashfile.exp new file mode 100644 index 000000000..ff1e8293e --- /dev/null +++ b/tests/lang/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/lang/eval-okay-hashfile.nix new file mode 100644 index 000000000..aff5a1856 --- /dev/null +++ b/tests/lang/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/lang/eval-okay-hashstring.exp new file mode 100644 index 000000000..d720a082d --- /dev/null +++ b/tests/lang/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/tests/lang/eval-okay-hash.nix b/tests/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hash.nix rename to tests/lang/eval-okay-hashstring.nix From 201f92e02c3a5d58d27ab9a7aca3b76d56d18264 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 1 May 2019 11:38:48 +0200 Subject: [PATCH 0178/2548] Fixed Flake data type and flake fetching --- src/libexpr/primops/flake.cc | 110 +++++++++++++++-------------------- src/libexpr/primops/flake.hh | 30 ++++++---- src/nix/flake.cc | 55 ++++++++++++------ 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index ab797fe01..89267e2b1 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -204,28 +204,32 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const return flakeRef; } -static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool impureIsAllowed = false) +// Lookups happen here too +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - if (evalSettings.pureEval && !impureIsAllowed && !fRef.isImmutable()) - throw Error("requested to fetch mutable flake '%s' in pure mode", fRef); + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + + if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); // This only downloads only one revision of the repo, not the entire history. - if (auto refData = std::get_if(&fRef.data)) { + if (auto refData = std::get_if(&resolvedRef.data)) { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", refData->owner, refData->repo, - fRef.rev ? fRef.rev->to_string(Base16, false) - : fRef.ref ? *fRef.ref : "master"); + resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) + : resolvedRef.ref ? *resolvedRef.ref : "master"); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") url += "?access_token=" + accessToken; auto result = getDownloader()->downloadCached(state.store, url, true, "source", - Hash(), nullptr, fRef.rev ? 1000000000 : settings.tarballTtl); + Hash(), nullptr, resolvedRef.rev ? 1000000000 : settings.tarballTtl); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); @@ -233,72 +237,60 @@ static FlakeSourceInfo fetchFlake(EvalState & state, const FlakeRef fRef, bool i if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - FlakeSourceInfo info(fRef); + std::string rev = std::string(*result.etag, 1, result.etag->size() - 2); + const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + rev); + SourceInfo info(ref); info.storePath = result.path; - info.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); - info.flakeRef.rev = info.rev; - info.flakeRef.ref = {}; return info; } // This downloads the entire git history - else if (auto refData = std::get_if(&fRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, fRef.ref, fRef.rev, "source"); - FlakeSourceInfo info(fRef); + else if (auto refData = std::get_if(&resolvedRef.data)) { + auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); + const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + SourceInfo info(ref); info.storePath = gitInfo.storePath; - info.rev = gitInfo.rev; info.revCount = gitInfo.revCount; - info.flakeRef.ref = gitInfo.ref; - info.flakeRef.rev = info.rev; return info; } - else if (auto refData = std::get_if(&fRef.data)) { + else if (auto refData = std::get_if(&resolvedRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); - FlakeSourceInfo info(fRef); + const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + SourceInfo info(ref); info.storePath = gitInfo.storePath; - info.rev = gitInfo.rev; info.revCount = gitInfo.revCount; - info.flakeRef.ref = gitInfo.ref; - info.flakeRef.rev = info.rev; return info; } else abort(); } -// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within this function. +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { - FlakeRef resolvedRef = lookupFlake(state, flakeRef, - impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - FlakeSourceInfo sourceInfo = fetchFlake(state, resolvedRef, impureIsAllowed); - debug("got flake source '%s' with revision %s", - sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + FlakeRef resolvedRef = sourceInfo.resolvedRef; - resolvedRef = sourceInfo.flakeRef; // `resolvedRef` is now immutable + resolvedRef = sourceInfo.resolvedRef; // `resolvedRef` is now immutable state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(sourceInfo.storePath); - Flake flake(resolvedRef, std::move(sourceInfo)); - if (std::get_if(&resolvedRef.data)) { - // FIXME: ehm? - if (flake.sourceInfo.rev) - flake.ref = FlakeRef(resolvedRef.baseRef().to_string() - + "/" + flake.sourceInfo.rev->to_string(Base16, false)); - } - // Guard against symlink attacks. - auto flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); if (!isInDir(flakeFile, sourceInfo.storePath)) throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + + Flake flake(flakeRef, sourceInfo); + if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); @@ -344,24 +336,18 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe // Get the `NonFlake` corresponding to a `FlakeRef`. NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias) { - FlakeSourceInfo sourceInfo = fetchFlake(state, flakeRef); - debug("got non-flake source '%s' with revision %s", - sourceInfo.storePath, sourceInfo.rev.value_or(Hash(htSHA1)).to_string(Base16, false)); + SourceInfo sourceInfo = fetchFlake(state, flakeRef); + debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - auto flakePath = sourceInfo.storePath; - state.store->assertStorePath(flakePath); + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + NonFlake nonFlake(flakeRef, sourceInfo); + + nonFlake.storePath = sourceInfo.storePath; + state.store->assertStorePath(nonFlake.storePath); if (state.allowedPaths) - state.allowedPaths->insert(flakePath); - - NonFlake nonFlake(flakeRef); - if (std::get_if(&flakeRef.data)) { - if (sourceInfo.rev) - nonFlake.ref = FlakeRef(flakeRef.baseRef().to_string() - + "/" + sourceInfo.rev->to_string(Base16, false)); - } - - nonFlake.path = flakePath; + state.allowedPaths->insert(nonFlake.storePath); nonFlake.alias = alias; @@ -380,7 +366,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile lockFile; if (isTopFlake) - lockFile = readLockFile(flake.sourceInfo.storePath + "/" + flake.ref.subdir + "/flake.lock"); // FIXME: symlink attack + lockFile = readLockFile(flake.storePath + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack ResolvedFlake deps(flake); @@ -399,13 +385,13 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - LockFile::FlakeEntry entry(resolvedFlake.flake.sourceInfo.flakeRef); + LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef); for (auto & newResFlake : resolvedFlake.flakeDeps) - entry.flakeEntries.insert_or_assign(newResFlake.flake.id, dependenciesToFlakeEntry(newResFlake)); + entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake)); for (auto & nonFlake : resolvedFlake.nonFlakeDeps) - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.ref); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.resolvedRef); return entry; } @@ -453,18 +439,18 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) auto vNonFlake = state.allocAttr(v, nonFlake.alias); state.mkAttrs(*vNonFlake, 4); - state.store->isValidPath(nonFlake.path); - mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.path, {nonFlake.path}); + state.store->isValidPath(nonFlake.storePath); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.storePath, {nonFlake.storePath}); } mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); - auto & path = resFlake.flake.sourceInfo.storePath; + auto & path = resFlake.flake.storePath; state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); - if (resFlake.flake.sourceInfo.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); + if (resFlake.flake.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.revCount); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *resFlake.flake.vProvides, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 6329c36ec..f3bf76cc7 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -23,7 +23,7 @@ struct LockFile { FlakeRef ref; std::map flakeEntries; - std::map nonFlakeEntries; + std::map nonFlakeEntries; FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; }; @@ -43,37 +43,43 @@ std::shared_ptr readRegistry(const Path &); void writeRegistry(const FlakeRegistry &, const Path &); -struct FlakeSourceInfo +struct SourceInfo { - FlakeRef flakeRef; + FlakeRef resolvedRef; Path storePath; - std::optional rev; std::optional revCount; // date - FlakeSourceInfo(const FlakeRef & flakeRef) : flakeRef(flakeRef) { } + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; }; struct Flake { FlakeId id; - FlakeRef ref; + FlakeRef originalRef; + FlakeRef resolvedRef; std::string description; - FlakeSourceInfo sourceInfo; + std::optional revCount; + Path storePath; std::vector requires; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc - Flake(const FlakeRef & flakeRef, FlakeSourceInfo && sourceInfo) - : ref(flakeRef), sourceInfo(sourceInfo) {}; + // date + // content hash + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), + resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; }; struct NonFlake { FlakeAlias alias; - FlakeRef ref; - Path path; + FlakeRef originalRef; + FlakeRef resolvedRef; + std::optional revCount; + Path storePath; // date // content hash - NonFlake(const FlakeRef flakeRef) : ref(flakeRef) {}; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), + resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; }; std::shared_ptr getGlobalRegistry(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e38c4db0b..0af368570 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -45,35 +45,53 @@ void printFlakeInfo(Flake & flake, bool json) { if (json) { nlohmann::json j; j["id"] = flake.id; - j["uri"] = flake.sourceInfo.flakeRef.to_string(); + j["uri"] = flake.resolvedRef.to_string(); j["description"] = flake.description; - if (flake.sourceInfo.rev) - j["revision"] = flake.sourceInfo.rev->to_string(Base16, false); - if (flake.sourceInfo.revCount) - j["revCount"] = *flake.sourceInfo.revCount; - j["path"] = flake.sourceInfo.storePath; + if (flake.resolvedRef.ref) + j["branch"] = *flake.resolvedRef.ref; + if (flake.resolvedRef.rev) + j["revision"] = flake.resolvedRef.rev->to_string(Base16, false); + if (flake.revCount) + j["revCount"] = *flake.revCount; + j["path"] = flake.storePath; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; - std::cout << "URI: " << flake.sourceInfo.flakeRef << "\n"; + std::cout << "URI: " << flake.resolvedRef.to_string() << "\n"; std::cout << "Description: " << flake.description << "\n"; - if (flake.sourceInfo.rev) - std::cout << "Revision: " << flake.sourceInfo.rev->to_string(Base16, false) << "\n"; - if (flake.sourceInfo.revCount) - std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; - std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; + if (flake.resolvedRef.ref) + std::cout << "Branch: " << *flake.resolvedRef.ref; + if (flake.resolvedRef.rev) + std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n"; + if (flake.revCount) + std::cout << "Revcount: " << *flake.revCount << "\n"; + std::cout << "Path: " << flake.storePath << "\n"; } } void printNonFlakeInfo(NonFlake & nonFlake, bool json) { if (json) { nlohmann::json j; - j["name"] = nonFlake.alias; - j["location"] = nonFlake.path; + j["id"] = nonFlake.alias; + j["uri"] = nonFlake.resolvedRef.to_string(); + if (nonFlake.resolvedRef.ref) + j["branch"] = *nonFlake.resolvedRef.ref; + if (nonFlake.resolvedRef.rev) + j["revision"] = nonFlake.resolvedRef.rev->to_string(Base16, false); + if (nonFlake.revCount) + j["revCount"] = *nonFlake.revCount; + j["path"] = nonFlake.storePath; std::cout << j.dump(4) << std::endl; } else { - std::cout << "name: " << nonFlake.alias << "\n"; - std::cout << "Location: " << nonFlake.path << "\n"; + std::cout << "ID: " << nonFlake.alias << "\n"; + std::cout << "URI: " << nonFlake.resolvedRef.to_string() << "\n"; + if (nonFlake.resolvedRef.ref) + std::cout << "Branch: " << *nonFlake.resolvedRef.ref; + if (nonFlake.resolvedRef.rev) + std::cout << "Revision: " << nonFlake.resolvedRef.rev->to_string(Base16, false) << "\n"; + if (nonFlake.revCount) + std::cout << "Revcount: " << *nonFlake.revCount << "\n"; + std::cout << "Path: " << nonFlake.storePath << "\n"; } } @@ -244,14 +262,13 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs FlakeRegistry userRegistry = *readRegistry(userRegistryPath); auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - it->second = getFlake(*evalState, it->second, true).ref; - // The 'ref' in 'flake' is immutable. + it->second = getFlake(*evalState, it->second, true).resolvedRef; writeRegistry(userRegistry, userRegistryPath); } else { std::shared_ptr globalReg = getGlobalRegistry(); it = globalReg->entries.find(FlakeRef(alias)); if (it != globalReg->entries.end()) { - FlakeRef newRef = getFlake(*evalState, it->second, true).ref; + FlakeRef newRef = getFlake(*evalState, it->second, true).resolvedRef; userRegistry.entries.insert_or_assign(alias, newRef); writeRegistry(userRegistry, userRegistryPath); } else From c38c726eb5d447c7e9d894d57cd05ac46c173ddd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 21:15:45 +0200 Subject: [PATCH 0179/2548] Fix test failures when $TMPDIR changes --- tests/build-dry.sh | 6 +++--- tests/nix-copy-ssh.sh | 2 +- tests/nix-shell.sh | 8 ++++---- tests/placeholders.sh | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c..e72533e70 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a4..eb801548d 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399..ee502dddb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc..cd1bb7bc2 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY From 3c171851a8aed22e977a1798dc5d306faa6e5b63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 22:29:16 +0200 Subject: [PATCH 0180/2548] Make the URL/path of the global flake registry configurable --- src/libexpr/eval.cc | 8 ++++++++ src/libexpr/eval.hh | 5 +++++ src/libexpr/primops/flake.cc | 3 +-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4bd1280ad..2d83af983 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1972,6 +1972,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings evalSettings; +EvalSettings::EvalSettings() +{ + if (flakeRegistry == "") + // FIXME: static initialization order fiasco. But this will go + // away when we switch to an online registry. + flakeRegistry = settings.nixDataDir + "/nix/flake-registry.json"; +} + static GlobalConfig::Register r1(&evalSettings); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 44988cd70..b0bf777fc 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -367,6 +367,11 @@ struct EvalSettings : Config Setting allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting flakeRegistry{this, "", "flake-registry", + "Path or URI of the global flake registry."}; + + EvalSettings(); }; extern EvalSettings evalSettings; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 89267e2b1..f73ab4424 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -126,8 +126,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path) std::shared_ptr getGlobalRegistry() { - Path registryFile = settings.nixDataDir + "/nix/flake-registry.json"; - return readRegistry(registryFile); + return readRegistry(evalSettings.flakeRegistry); } Path getUserRegistryPath() From 2a41a567e29846cf32d38f338b992069592770c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:06:15 +0200 Subject: [PATCH 0181/2548] Improve FlakeRef::to_string() We were incorrectly using path syntax (i.e. //) for Git repositories. This is only valid for GitHub flakerefs. --- src/libexpr/primops/flakeref.cc | 42 +++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 141d61c0d..784a0868e 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -147,31 +147,53 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::string FlakeRef::to_string() const { std::string string; + bool first = true; - if (auto refData = std::get_if(&data)) + auto addParam = + [&](const std::string & name, std::string value) { + string += first ? '?' : '&'; + first = false; + string += name; + string += '='; + string += value; // FIXME: escaping + }; + + if (auto refData = std::get_if(&data)) { string = refData->alias; + if (ref) string += '/' + *ref; + if (rev) string += '/' + rev->to_string(Base16, false); + } + + else if (auto refData = std::get_if(&data)) { + assert(subdir == ""); + assert(!rev); + assert(!ref); + return refData->path; + } else if (auto refData = std::get_if(&data)) { assert(!(ref && rev)); string = "github:" + refData->owner + "/" + refData->repo; + if (ref) { string += '/'; string += *ref; } + if (rev) { string += '/'; string += rev->to_string(Base16, false); } + if (subdir != "") addParam("dir", subdir); } else if (auto refData = std::get_if(&data)) { assert(!rev || ref); string = refData->uri; + + if (ref) { + addParam("ref", *ref); + if (rev) + addParam("rev", rev->to_string(Base16, false)); + } + + if (subdir != "") addParam("dir", subdir); } - else if (auto refData = std::get_if(&data)) - return refData->path; - else abort(); - // FIXME: need to use ?rev etc. for IsGit URIs. - string += (ref ? "/" + *ref : "") + - (rev ? "/" + rev->to_string(Base16, false) : ""); - - if (subdir != "") string += "?dir=" + subdir; - return string; } From ddd42b7e948acc1b541741250c9a77e86aaada04 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:20:42 +0200 Subject: [PATCH 0182/2548] Fix immutable flakeref construction We were appending ref/revs incorrectly for the IsGit case (by appending // rather than ?ref=&rev=size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - std::string rev = std::string(*result.etag, 1, result.etag->size() - 2); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + rev); + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); SourceInfo info(ref); info.storePath = result.path; @@ -247,7 +247,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool // This downloads the entire git history else if (auto refData = std::get_if(&resolvedRef.data)) { auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; @@ -258,7 +260,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); - const FlakeRef ref(resolvedRef.baseRef().to_string() + "/" + gitInfo.ref + "/" + gitInfo.rev.to_string(Base16, false)); + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; From 2d5a21968842c88b425e1a591bd413c484a470e7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 23:22:47 +0200 Subject: [PATCH 0183/2548] Add basic flake tests --- Makefile.config.in | 1 + configure.ac | 1 + release-common.nix | 1 + tests/{config.nix => config.nix.in} | 2 +- tests/flakes.sh | 117 ++++++++++++++++++++++++++++ tests/local.mk | 8 +- 6 files changed, 126 insertions(+), 4 deletions(-) rename tests/{config.nix => config.nix.in} (92%) create mode 100644 tests/flakes.sh diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646..f2273eaed 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -35,6 +35,7 @@ prefix = @prefix@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ +system = @system@ doc_generate = @doc_generate@ xmllint = @xmllint@ xsltproc = @xsltproc@ diff --git a/configure.ac b/configure.ac index f5b1614f1..8087f308b 100644 --- a/configure.ac +++ b/configure.ac @@ -129,6 +129,7 @@ NEED_PROG(gzip, gzip) NEED_PROG(xz, xz) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) +NEED_PROG(jq, jq) NEED_PROG(cat, cat) diff --git a/release-common.nix b/release-common.nix index 4c5565985..b745932a7 100644 --- a/release-common.nix +++ b/release-common.nix @@ -56,6 +56,7 @@ rec { # Tests git mercurial + jq ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/tests/config.nix b/tests/config.nix.in similarity index 92% rename from tests/config.nix rename to tests/config.nix.in index 6ba91065b..ff5aeb31a 100644 --- a/tests/config.nix +++ b/tests/config.nix.in @@ -5,7 +5,7 @@ rec { path = coreutils; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; diff --git a/tests/flakes.sh b/tests/flakes.sh new file mode 100644 index 000000000..26ffd6a9f --- /dev/null +++ b/tests/flakes.sh @@ -0,0 +1,117 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping flake tests" + exit 99 +fi + +clearStore + +registry=$TEST_ROOT/registry.json + +flake1=$TEST_ROOT/flake1 +flake2=$TEST_ROOT/flake2 +flake3=$TEST_ROOT/flake3 + +for repo in $flake1 $flake2 $flake3; do + rm -rf $repo + mkdir $repo + git -C $repo init + git -C $repo config user.email "foobar@example.com" + git -C $repo config user.name "Foobar" +done + +cat > $flake1/flake.nix < $flake2/flake.nix < $registry < Date: Tue, 7 May 2019 17:32:01 -0500 Subject: [PATCH 0184/2548] fix hashfile test that wasn't failing due to eval laziness See: https://github.com/NixOS/nix/commit/7becb1bf1c2ec1544a5374580a97b36273506baf#r33450554 --- tests/lang/eval-fail-hashfile-missing.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/lang/eval-fail-hashfile-missing.nix index 42fb1ec7e..ce098b823 100644 --- a/tests/lang/eval-fail-hashfile-missing.nix +++ b/tests/lang/eval-fail-hashfile-missing.nix @@ -1,5 +1,5 @@ let paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; in - builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) + toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) From 9d07c3717b0b5accb3923ce0920e72fb26617468 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 13:24:37 +0200 Subject: [PATCH 0185/2548] updateLockFile: Do "git add" in a slightly nicer way "--intent-to-add" ensures the change shows up in "git diff". --- src/libexpr/primops/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 89267e2b1..b451afba4 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -417,7 +417,7 @@ void updateLockFile(EvalState & state, const FlakeUri & flakeUri) // Hack: Make sure that flake.lock is visible to Git. Otherwise, // exportGit will fail to copy it to the Nix store. runProgram("git", true, - { "-C", refData->path, "add", + { "-C", refData->path, "add", "--intent-to-add", (flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" }); } else throw Error("flakeUri %s can't be updated because it is not a path", flakeUri); From 0d1c2e5baed2d75c02daa66962bc7863976978fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 13:26:05 +0200 Subject: [PATCH 0186/2548] Add Hydra flake to the registry --- flake-registry.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake-registry.json b/flake-registry.json index ae94f1df2..c6a549f92 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -8,6 +8,9 @@ }, "nixpkgs": { "uri": "github:edolstra/nixpkgs/release-19.03" + }, + "hydra": { + "uri": "github:NixOS/hydra/flake" } }, "version": 1 From 77e1f9010c5328bee16d85661fe359746cb70e40 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 13:38:32 +0200 Subject: [PATCH 0187/2548] Export missing rev/shortRev attributes --- src/libexpr/primops/flake.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index b451afba4..dc8df7f6f 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -441,6 +441,8 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) state.store->isValidPath(nonFlake.storePath); mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.storePath, {nonFlake.storePath}); + + // FIXME: add rev, shortRev, revCount, ... } mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); @@ -449,6 +451,13 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); + if (resFlake.flake.resolvedRef.rev) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), + resFlake.flake.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), + resFlake.flake.resolvedRef.rev->gitShortRev()); + } + if (resFlake.flake.revCount) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.revCount); From a887892eb607d45de47e77aeb1a756f7672395b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 14:17:32 +0200 Subject: [PATCH 0188/2548] nix-shell: Don't fail if run from a path containing the string "nix-shell" --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 618895d38..5b8e816d8 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -106,7 +106,7 @@ static void _main(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { + if (runEnv && argc > 1 && !std::regex_search(baseNameOf(argv[1]), std::regex("nix-shell"))) { script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); From 7c6391ddc730519a632cc0ee526c94a04812d871 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 14:18:57 +0200 Subject: [PATCH 0189/2548] nix-shell: Don't absolutize '-p' expressions This prevents spurious syscalls like 25011 lstat("/home/eelco/with import { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ (hello) ]; } \"\"", 0x7ffe9c67f580) = -1 ENOENT (No such file or directory) --- src/nix-build/nix-build.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 618895d38..c6a4d4166 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -274,19 +274,21 @@ static void _main(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : left) { - auto absolute = i; - try { - absolute = canonPath(absPath(i), true); - } catch (Error e) {}; if (fromArgs) exprs.push_back(state->parseExprFromString(i, absPath("."))); - else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error e) {}; + if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + } } /* Evaluate them into derivations. */ From cb5ebc5c1120da8900ef074ed300685cc3177ae6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 17:07:35 +0200 Subject: [PATCH 0190/2548] nix dev-shell: Keep $TERM --- src/nix/shell.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 0813d122c..8b9106171 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -133,6 +133,7 @@ struct Common : InstallableCommand "SHLVL", "TEMP", "TEMPDIR", + "TERM", "TMP", "TMPDIR", "TZ", From 455aa8d9ea55d3ea661b3c6f93e3ed5a43a82746 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 18:20:35 +0200 Subject: [PATCH 0191/2548] Add newline at end of lockfile Suggested by @grahamc. --- flake.lock | 2 +- src/libexpr/primops/flake.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 9848cb996..0a5f76db7 100644 --- a/flake.lock +++ b/flake.lock @@ -6,4 +6,4 @@ } }, "version": 1 -} \ No newline at end of file +} diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 5e732b362..1d409105f 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -121,7 +121,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path) for (auto & x : lockFile.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); createDirs(dirOf(path)); - writeFile(path, json.dump(4)); // '4' = indentation in json file + writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file } std::shared_ptr getGlobalRegistry() From 8fc1c3f413c20b4519a41e38365aa71fa06a5aae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 22:08:38 +0200 Subject: [PATCH 0192/2548] Fix assertion failure in FlakeRef::to_string() Fixes $ nix build nix: src/libexpr/primops/flakeref.cc:169: std::__cxx11::string nix::FlakeRef::to_string() const: Assertion `!rev' failed. Aborted e.g. when flake.nix doesn't exist. Also use gitRev(). --- src/libexpr/primops/flakeref.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 784a0868e..3842b3f1a 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -161,13 +161,13 @@ std::string FlakeRef::to_string() const if (auto refData = std::get_if(&data)) { string = refData->alias; if (ref) string += '/' + *ref; - if (rev) string += '/' + rev->to_string(Base16, false); + if (rev) string += '/' + rev->gitRev(); } else if (auto refData = std::get_if(&data)) { assert(subdir == ""); - assert(!rev); - assert(!ref); + if (ref) addParam("ref", *ref); + if (rev) addParam("rev", rev->gitRev()); return refData->path; } @@ -175,7 +175,7 @@ std::string FlakeRef::to_string() const assert(!(ref && rev)); string = "github:" + refData->owner + "/" + refData->repo; if (ref) { string += '/'; string += *ref; } - if (rev) { string += '/'; string += rev->to_string(Base16, false); } + if (rev) { string += '/'; string += rev->gitRev(); } if (subdir != "") addParam("dir", subdir); } @@ -186,7 +186,7 @@ std::string FlakeRef::to_string() const if (ref) { addParam("ref", *ref); if (rev) - addParam("rev", rev->to_string(Base16, false)); + addParam("rev", rev->gitRev()); } if (subdir != "") addParam("dir", subdir); From cead210e664785b25d7b5819133b74b4cc2c392b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 22:40:21 +0200 Subject: [PATCH 0193/2548] Add patchelf to the flake registry --- flake-registry.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake-registry.json b/flake-registry.json index c6a549f92..348bd010a 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -11,6 +11,9 @@ }, "hydra": { "uri": "github:NixOS/hydra/flake" + }, + "patchelf": { + "uri": "github:NixOS/patchelf" } }, "version": 1 From a746dc64d2fa45456817c6ef8a4f82c6217e0392 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 May 2019 13:55:33 +0200 Subject: [PATCH 0194/2548] Add currently failing test for #81 --- tests/flakes.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 26ffd6a9f..da952d552 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -110,6 +110,10 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar [[ -e $flake2/flake.lock ]] git -C $flake2 commit flake.lock -m 'Add flake.lock' +# Rerunning the build should not change the lockfile. +nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar +[[ -z $(git -C $flake2 diff) ]] + # Now we should be able to build the flake in pure mode. nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar From 391e1f511d90461dcdd52e495e2596bbd2ea3c5a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 May 2019 13:59:50 +0200 Subject: [PATCH 0195/2548] Add test for indirect flake dependencies I.e. flake3 depends on flake2 which depends on flake1. Currently this fails with error: indirect flake reference 'flake1' is not allowed because we're not propagating lockfiles downwards properly. --- tests/flakes.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index da952d552..7b827a418 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -59,6 +59,25 @@ EOF git -C $flake2 add flake.nix git -C $flake2 commit -m 'Initial' +cat > $flake3/flake.nix < $registry < Date: Wed, 1 May 2019 17:01:03 +0200 Subject: [PATCH 0196/2548] Fixed issue #47 Content hashes --- src/libexpr/primops/flake.cc | 31 +++++++++++++++++++++---------- src/libexpr/primops/flake.hh | 17 +++++++++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 1d409105f..e9896d665 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -50,7 +50,8 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) if (!flakeRef.isImmutable()) throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); - LockFile::FlakeEntry entry(flakeRef); + Hash hash = Hash((std::string) json["contentHash"]); + LockFile::FlakeEntry entry(flakeRef, hash); auto nonFlakeRequires = json["nonFlakeRequires"]; @@ -58,7 +59,9 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - entry.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + Hash hash = Hash((std::string) i->value("contentHash", "")); + LockFile::NonFlakeEntry newEntry(flakeRef, hash); + entry.nonFlakeEntries.insert_or_assign(i.key(), newEntry); } auto requires = json["requires"]; @@ -86,9 +89,10 @@ LockFile readLockFile(const Path & path) for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); + LockFile::NonFlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"])); if (!flakeRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), flakeRef); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), entry); } auto requires = json["requires"]; @@ -103,8 +107,11 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) { nlohmann::json json; json["uri"] = entry.ref.to_string(); - for (auto & x : entry.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + json["contentHash"] = entry.contentHash.to_string(SRI); + for (auto & x : entry.nonFlakeEntries) { + json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI); + } for (auto & x : entry.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); return json; @@ -115,8 +122,10 @@ void writeLockFile(const LockFile & lockFile, const Path & path) nlohmann::json json; json["version"] = 1; json["nonFlakeRequires"] = nlohmann::json::object(); - for (auto & x : lockFile.nonFlakeEntries) - json["nonFlakeRequires"][x.first]["uri"] = x.second.to_string(); + for (auto & x : lockFile.nonFlakeEntries) { + json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI); + } json["requires"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); @@ -293,6 +302,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); Flake flake(flakeRef, sourceInfo); + flake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash; if (!pathExists(flakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); @@ -346,12 +356,13 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al NonFlake nonFlake(flakeRef, sourceInfo); - nonFlake.storePath = sourceInfo.storePath; state.store->assertStorePath(nonFlake.storePath); if (state.allowedPaths) state.allowedPaths->insert(nonFlake.storePath); + nonFlake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash; + nonFlake.alias = alias; return nonFlake; @@ -388,13 +399,13 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef); + LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash); for (auto & newResFlake : resolvedFlake.flakeDeps) entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake)); for (auto & nonFlake : resolvedFlake.nonFlakeDeps) - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonFlake.resolvedRef); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, LockFile::NonFlakeEntry(nonFlake.resolvedRef, nonFlake.hash)); return entry; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index f3bf76cc7..691f68b0a 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -19,16 +19,24 @@ struct FlakeRegistry struct LockFile { + struct NonFlakeEntry + { + FlakeRef ref; + Hash contentHash; + NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; + }; + struct FlakeEntry { FlakeRef ref; + Hash contentHash; std::map flakeEntries; - std::map nonFlakeEntries; - FlakeEntry(const FlakeRef & flakeRef) : ref(flakeRef) {}; + std::map nonFlakeEntries; + FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; }; std::map flakeEntries; - std::map nonFlakeEntries; + std::map nonFlakeEntries; }; typedef std::vector> Registries; @@ -60,6 +68,7 @@ struct Flake std::string description; std::optional revCount; Path storePath; + Hash hash; // content hash std::vector requires; std::map nonFlakeRequires; Value * vProvides; // FIXME: gc @@ -76,8 +85,8 @@ struct NonFlake FlakeRef resolvedRef; std::optional revCount; Path storePath; + Hash hash; // content hash // date - // content hash NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; }; From 68b17ef731570db1d048e26a9a9ab8b8c3f09e0c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 May 2019 15:08:09 +0200 Subject: [PATCH 0197/2548] Update flake.lock --- flake.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.lock b/flake.lock index 0a5f76db7..965d038ce 100644 --- a/flake.lock +++ b/flake.lock @@ -2,6 +2,7 @@ "nonFlakeRequires": {}, "requires": { "nixpkgs": { + "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" } }, From 5a80cccc7075de151c79a818bad758e217cf1e9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 01:48:33 +0200 Subject: [PATCH 0198/2548] Hack to fix tests.evalNixpkgs --- release.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index a47ca862f..f98e6d6ed 100644 --- a/release.nix +++ b/release.nix @@ -268,7 +268,8 @@ let tests.evalNixpkgs = import (nixpkgs + "/pkgs/top-level/make-tarball.nix") { - inherit nixpkgs; + # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount. + nixpkgs = nixpkgs // { revCount = 0; }; inherit pkgs; nix = build.x86_64-linux; officialRelease = false; From 156e3a9daacce48f2f53939dee92dd063dfc507a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 01:50:28 +0200 Subject: [PATCH 0199/2548] nix dev-shell: Ignore SSL_CERT_FILE --- src/nix/shell.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 8b9106171..2ccad930f 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -131,6 +131,7 @@ struct Common : InstallableCommand "PWD", "SHELLOPTS", "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", "TERM", From 6ade7ec022c836b7d1f9bd06be45e2c07835ec8c Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 6 May 2019 22:23:15 +0200 Subject: [PATCH 0200/2548] progress-bar: hide expected if expected is 0 (unknown) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes, "expected" can be "0", but in fact means "unknown". This is for example the case when downloading a file while the http server doesn't send the `Content-Length` header, like when running `nix build` pointing to a nixpkgs checkout streamed from GitHub: ⇒ nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello [1.8/0.0 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz' In that case, don't show that weird progress bar, but only the (slowly increasing) downloaded size ("done"). ⇒ nix build -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz hello [1.8 MiB DL] downloading 'https://github.com/NixOS/nixpkgs/archive/master.tar.gz' This commit also updates fmt calls with three numbers (when something is currently 'running' too) - I'm not sure if this can be provoked, but showing "0" as expected doesn't make any sense, as we're obviously doing more than nothing. --- src/nix/progress-bar.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3..8da72bc94 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -333,11 +333,18 @@ public: if (running || done || expected || failed) { if (running) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); else if (expected != done) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); else s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); s = fmt(itemFmt, s); From ff6867ab94cbe9ddcb4ba18d68a4a2dcb79b865d Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 May 2019 15:53:40 +0200 Subject: [PATCH 0201/2548] build: move needsHashRewrite initialization to startBuilder The value of useChroot is not set yet in the constructor, resulting in hash rewriting being enabled in certain cases where it should not be. Fixes #2801 --- src/libstore/build.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 91eb97dfb..30825add4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -997,13 +997,6 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut , wantedOutputs(wantedOutputs) , buildMode(buildMode) { -#if __linux__ - needsHashRewrite = !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - needsHashRewrite = true; -#endif - state = &DerivationGoal::getDerivation; name = (format("building of '%1%'") % drvPath).str(); trace("created"); @@ -1852,6 +1845,13 @@ void DerivationGoal::startBuilder() #endif } +#if __linux__ + needsHashRewrite = !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + needsHashRewrite = true; +#endif + /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ if (settings.buildUsersGroup != "" && getuid() == 0) { From d75bdb5793e5ebf9e480f5a0012d141347725801 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sun, 12 May 2019 16:46:21 +0200 Subject: [PATCH 0202/2548] build: add test for sandboxed --check --- tests/linux-sandbox.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index acfd46c54..52967d07d 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -25,3 +25,6 @@ nix path-info -r $outPath | grep input-2 nix ls-store -R -l $outPath | grep foobar nix cat-store $outPath/foobar | grep FOOBAR + +# Test --check without hash rewriting. +nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store From c78686e411e0a14cff51836fe6c35d7584171df3 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 May 2019 16:39:31 -0400 Subject: [PATCH 0203/2548] build: run diff-hook under --check and document diff-hook --- .../advanced-topics/advanced-topics.xml | 1 + doc/manual/advanced-topics/diff-hook.xml | 207 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 81 ++++++- src/libstore/build.cc | 30 ++- 4 files changed, 303 insertions(+), 16 deletions(-) create mode 100644 doc/manual/advanced-topics/diff-hook.xml diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index b710f9f2b..c304367aa 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,5 +7,6 @@ Advanced Topics + diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 000000000..d2613f6df --- /dev/null +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,207 @@ + + +Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option> + +Check build reproducibility by running builds multiple times +and comparing their results. + +Specify a program with Nix's to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same. + +For purposes of demonstration, we'll use the following Nix file, +deterministic.nix for testing: + + +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} + + +Additionally, nix.conf contains: + + +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true + + +where /etc/nix/my-diff-hook is an executable +file containing: + + +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2" + + + + The diff hook can be run as root. Take care to run as little + as possible as root, for this example we use runuser + to drop privileges. + + + + +
+ + Spot-Checking Build Determinism + + + + Verify a path which already exists in the Nix store by passing + to the build command. + + + If the build passes and is deterministic, Nix will exit with a + status code of 0: + + +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + + + If the build is not deterministic, Nix will exit with a status + code of 1: + + +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs + + +In the Nix daemon's log, we will now see: + +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 + + + + Using with + will cause Nix to keep the second build's output in a special, + .check path: + + +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' + + + In particular, notice the + /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check + output. Nix has copied the build results to that directory where you + can examine it. + + + <literal>.check</literal> paths are not registered store paths + + Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection. + + The path is guaranteed to be alive for the duration of + 's execution, but may be deleted + any time after. + + If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist. + + + + is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible + + + Run the build without , and then try with + again. + +
+ +
+ + Automatic and Optionally Enforced Determinism Verification + + + + Automatically verify every build at build time by executing the + build multiple times. + + + + Setting and + in your + nix.conf permits the automated verification + of every build Nix performs. + + + + The following configuration will run each build three times, and + will require the build to be deterministic: + + +enforce-determinism = true +repeat = 2 + + + + + Setting to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + +enforce-determinism = false +repeat = 1 + + + + + An example output of this configuration: + +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable + + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index f0da1f612..a1a5d6e12 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -1,7 +1,9 @@ + + xml:id="sec-conf-file" + version="5"> nix.conf @@ -240,6 +242,64 @@ false. + diff-hook + + + Absolute path to an executable capable of diffing build results. + The hook executes if is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + + + + + The root user executes the diff hook in a daemonised + installation. See for + information on using the diff hook safely. + + + + The diff hook program receives three parameters: + + + + + A path to the previous build's results + + + + + + A path to the current build's results + + + + + + The path to the build's derivation + + + + + The diff hook should not print data to stderr or stdout, as + output is not displayed to the user. However, if information is + printed, it will be printed in the nix-daemon + log. + + When using the Nix daemon, diff-hook must + be set in the nix.conf configuration file, and + cannot be passed at the command line. + + + + + + enforce-determinism + + See . + + extra-sandbox-paths @@ -595,9 +655,9 @@ password my-password they are deterministic. The default value is 0. If the value is non-zero, every build is repeated the specified number of times. If the contents of any of the runs differs from the - previous ones, the build is rejected and the resulting store paths - are not registered as “valid” in Nix’s database. - + previous ones and is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database. require-sigs @@ -628,6 +688,19 @@ password my-password + run-diff-hook + + + If true, enable the execution of . + + + + When using the Nix daemon, run-diff-hook must + be set in the nix.conf configuration file, + and cannot be passed at the command line. + + + sandbox diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 91eb97dfb..026828c53 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,6 +461,19 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } +void handleDiffHook(Path tryA, Path tryB, Path drvPath) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } +} ////////////////////////////////////////////////////////////////////// @@ -3039,8 +3052,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.runDiffHook; - bool keepPreviousRound = settings.keepFailed || runDiffHook; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -3185,11 +3197,14 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + handleDiffHook(path, actualPath, drvPath); + if (settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else @@ -3254,16 +3269,7 @@ void DerivationGoal::registerOutputs() ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); - auto diffHook = settings.diffHook; - if (prevExists && diffHook != "" && runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {prev, i->second.path}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - } + handleDiffHook(prev, i->second.path, drvPath); if (settings.enforceDeterminism) throw NotDeterministic(msg); From 6df61db0600ca73ccd51e3e5bec5312a04e99da1 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 10 May 2019 20:59:39 -0400 Subject: [PATCH 0204/2548] diff hook: execute as the build user, and pass the temp dir --- doc/manual/advanced-topics/diff-hook.xml | 12 +++---- doc/manual/command-ref/conf-file.xml | 20 ++++++++---- src/libstore/build.cc | 41 +++++++++++++++++------- src/libutil/util.cc | 4 +-- src/libutil/util.hh | 2 ++ 5 files changed, 51 insertions(+), 28 deletions(-) diff --git a/doc/manual/advanced-topics/diff-hook.xml b/doc/manual/advanced-topics/diff-hook.xml index d2613f6df..fb4bf819f 100644 --- a/doc/manual/advanced-topics/diff-hook.xml +++ b/doc/manual/advanced-topics/diff-hook.xml @@ -46,17 +46,15 @@ file containing: #!/bin/sh exec >&2 echo "For derivation $3:" -/run/current-system/sw/bin/runuser -u nobody -- /run/current-system/sw/bin/diff -r "$1" "$2" +/run/current-system/sw/bin/diff -r "$1" "$2" - - The diff hook can be run as root. Take care to run as little - as possible as root, for this example we use runuser - to drop privileges. - - +The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built. +
Spot-Checking Build Determinism diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index a1a5d6e12..c5f90481b 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -252,13 +252,11 @@ false</literal>.</para> same. </para> - <warning> - <para> - The root user executes the diff hook in a daemonised - installation. See <xref linkend="chap-diff-hook" /> for - information on using the diff hook safely. - </para> - </warning> + <para> + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + </para> <para>The diff hook program receives three parameters:</para> @@ -280,6 +278,14 @@ false</literal>.</para> The path to the build's derivation </para> </listitem> + + <listitem> + <para> + The path to the build's scratch directory. This directory + will exist only if the build was run with + <option>--keep-failed</option>. + </para> + </listitem> </orderedlist> <para>The diff hook should not print data to stderr or stdout, as diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 026828c53..f38d2eaa0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,17 +461,26 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(Path tryA, Path tryB, Path drvPath) +void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } + auto wrapper = [&]() { + if (setgid(gid) == -1) + throw SysError("setgid failed"); + if (setuid(uid) == -1) + throw SysError("setuid failed"); + + try { + auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir}); + if (diff != "") + printError(chomp(diff)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + }; + + doFork(allowVfork, wrapper); } } @@ -3197,14 +3206,18 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { - handleDiffHook(path, actualPath, drvPath); - - if (settings.keepFailed) { + if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + handleDiffHook( + !buildUser && !drv->isBuiltin(), + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, drvPath, tmpDir); + throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else @@ -3269,7 +3282,11 @@ void DerivationGoal::registerOutputs() ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); - handleDiffHook(prev, i->second.path, drvPath); + handleDiffHook( + !buildUser && !drv->isBuiltin(), + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, i->second.path, drvPath, tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a71705665..0f4d3d92b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -914,8 +914,8 @@ void killUser(uid_t uid) /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function<void()> fun) +pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +pid_t doFork(bool allowVfork, std::function<void()> fun) { #ifdef __linux__ pid_t pid = allowVfork ? vfork() : fork(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 54936a5cb..824a35b98 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -265,6 +265,8 @@ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); +pid_t doFork(bool allowVfork, std::function<void()> fun); + struct RunOptions { Path program; From dde8eeb39ae9fb73011462c74e5fa6405e432147 Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sat, 11 May 2019 15:57:38 -0400 Subject: [PATCH 0205/2548] chdir, setgroups --- src/libstore/build.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f38d2eaa0..8397cd0d1 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -466,8 +466,12 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { auto wrapper = [&]() { + if (chdir("/") == -1) + throw SysError("chdir / failed"); if (setgid(gid) == -1) throw SysError("setgid failed"); + if (setgroups(0, 0) == -1) + throw SysError("setgroups failed"); if (setuid(uid) == -1) throw SysError("setuid failed"); From b4a05edbfe49f87555fd284dfb0d6c56ed43217d Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sat, 11 May 2019 16:35:53 -0400 Subject: [PATCH 0206/2548] runProgram: support gid, uid, chdir --- src/libstore/build.cc | 32 ++++++++++++++------------------ src/libutil/util.cc | 15 +++++++++++++-- src/libutil/util.hh | 5 +++-- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8397cd0d1..8902e22bd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -465,26 +465,22 @@ void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { - auto wrapper = [&]() { - if (chdir("/") == -1) - throw SysError("chdir / failed"); - if (setgid(gid) == -1) - throw SysError("setgid failed"); - if (setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (setuid(uid) == -1) - throw SysError("setuid failed"); + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; - try { - auto diff = runProgram(diffHook, true, {tryA, tryB, drvPath, tmpDir}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - }; + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); - doFork(allowVfork, wrapper); + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0f4d3d92b..55b9144f3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -16,6 +16,7 @@ #include <future> #include <fcntl.h> +#include <grp.h> #include <limits.h> #include <pwd.h> #include <sys/ioctl.h> @@ -914,8 +915,8 @@ void killUser(uid_t uid) /* Wrapper around vfork to prevent the child process from clobbering the caller's stack frame in the parent. */ -pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); -pid_t doFork(bool allowVfork, std::function<void()> fun) +static pid_t doFork(bool allowVfork, std::function<void()> fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function<void()> fun) { #ifdef __linux__ pid_t pid = allowVfork ? vfork() : fork(); @@ -1025,6 +1026,16 @@ void runProgram2(const RunOptions & options) if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); + //if (options.chdir && chdir((*options.chdir).c_str()) == -1) + // throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + Strings args_(options.args); args_.push_front(options.program); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 824a35b98..7c57d0afa 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -265,10 +265,11 @@ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); -pid_t doFork(bool allowVfork, std::function<void()> fun); - struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; Path program; bool searchPath = true; Strings args; From a5efe617862484ab7dd234a495d315e7b08aa519 Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 13:23:30 -0400 Subject: [PATCH 0207/2548] Clarify where output from the diff hook goes. --- doc/manual/command-ref/conf-file.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c5f90481b..24fbf28cf 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -288,10 +288,11 @@ false</literal>.</para> </listitem> </orderedlist> - <para>The diff hook should not print data to stderr or stdout, as - output is not displayed to the user. However, if information is - printed, it will be printed in the <command>nix-daemon</command> - log.</para> + <para> + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + </para> <para>When using the Nix daemon, <literal>diff-hook</literal> must be set in the <filename>nix.conf</filename> configuration file, and From 73b797c207e1c7a0fd9059d2cf1e3479502f8f1b Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 13:44:22 -0400 Subject: [PATCH 0208/2548] handleDiffHook: stop passing allowVfork --- src/libstore/build.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8902e22bd..b07461013 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -461,7 +461,7 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(bool allowVfork, uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { @@ -3213,7 +3213,6 @@ void DerivationGoal::registerOutputs() throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); handleDiffHook( - !buildUser && !drv->isBuiltin(), buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); @@ -3283,7 +3282,6 @@ void DerivationGoal::registerOutputs() : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); handleDiffHook( - !buildUser && !drv->isBuiltin(), buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), prev, i->second.path, drvPath, tmpDir); From ce02fc74b2db35e45906865c8a3ce2e98871eeb8 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan <daiderd@gmail.com> Date: Sun, 12 May 2019 22:47:41 +0200 Subject: [PATCH 0209/2548] build: make needsHashRewrite a method --- src/libstore/build.cc | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 30825add4..79dcdddbe 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -803,9 +803,6 @@ private: /* Whether we're currently doing a chroot build. */ bool useChroot = false; - /* Whether we need to perform hash rewriting if there are valid output paths. */ - bool needsHashRewrite; - Path chrootRootDir; /* RAII object to delete the chroot directory. */ @@ -885,6 +882,9 @@ public: Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + void timedOut() override; string key() override @@ -1037,6 +1037,17 @@ DerivationGoal::~DerivationGoal() } +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + void DerivationGoal::killChild() { if (pid != -1) { @@ -1845,13 +1856,6 @@ void DerivationGoal::startBuilder() #endif } -#if __linux__ - needsHashRewrite = !useChroot; -#else - /* Darwin requires hash rewriting even when sandboxing is enabled. */ - needsHashRewrite = true; -#endif - /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ if (settings.buildUsersGroup != "" && getuid() == 0) { @@ -2083,7 +2087,7 @@ void DerivationGoal::startBuilder() #endif } - if (needsHashRewrite) { + if (needsHashRewrite()) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -3067,7 +3071,7 @@ void DerivationGoal::registerOutputs() if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } - if (needsHashRewrite) { + if (needsHashRewrite()) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() From f1b8e9efe77014655f059b44afa05c38990dc4aa Mon Sep 17 00:00:00 2001 From: Graham Christensen <graham@grahamc.com> Date: Sun, 12 May 2019 17:03:01 -0400 Subject: [PATCH 0210/2548] runProgram: Uncomment chdir support --- src/libutil/util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 55b9144f3..17aee2d5c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1026,8 +1026,8 @@ void runProgram2(const RunOptions & options) if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); - //if (options.chdir && chdir((*options.chdir).c_str()) == -1) - // throw SysError("chdir failed"); + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); if (options.gid && setgid(*options.gid) == -1) throw SysError("setgid failed"); /* Drop all other groups if we're setgid. */ From 4c9ebd20d7d16501aaceecf584c15499f720bf65 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Wed, 15 May 2019 08:04:21 +0200 Subject: [PATCH 0211/2548] One FIXME was already fixed --- src/libexpr/primops/flake.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e9896d665..8dd9599bd 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -312,7 +312,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.forceAttrs(vInfo); - // FIXME: change to "id"? if (auto name = vInfo.attrs->get(state.sName)) flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); else From 696a98af5aa0f34b35fd2bbb714842495104ed13 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 09:02:48 +0200 Subject: [PATCH 0212/2548] Disable tests introduced by PR #82 This PR was not intended to be merged until those tests were actually passing. So disable them for now to unbreak the flakes branch. https://hydra.nixos.org/eval/1519271 --- tests/flakes.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 7b827a418..40ba42715 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -130,8 +130,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar git -C $flake2 commit flake.lock -m 'Add flake.lock' # Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar -[[ -z $(git -C $flake2 diff) ]] +#nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar +#[[ -z $(git -C $flake2 diff) ]] # Now we should be able to build the flake in pure mode. nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar @@ -140,4 +140,4 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar nix build -o $TEST_ROOT/result file://$flake2:bar # Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake3:xyzzy +#nix build -o $TEST_ROOT/result --flake-registry $registry $flake3:xyzzy From 3fd5425f948afa5122ff6b0aad60a6b961b57161 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 13:13:14 +0200 Subject: [PATCH 0213/2548] Fix shellcheck error https://hydra.nixos.org/build/93359951 --- scripts/install-nix-from-closure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 4dd249923..fc999d336 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -26,7 +26,7 @@ fi if [ "$(uname -s)" = "Darwin" ]; then macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) - if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" exit 1 fi From b6eb8a2d7e2ea8b083fdac15f537679ffe633183 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 14:30:09 +0200 Subject: [PATCH 0214/2548] nix-profile: Add all channels to $NIX_PATH Fixes #2709. --- scripts/nix-profile.sh.in | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index db03e16ba..f3cfa157c 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -51,10 +51,9 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then unset __nix_defexpr fi - # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that - # <nixpkgs> paths work when the user has fetched the Nixpkgs - # channel. - export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs> + # paths work when the user has fetched the Nixpkgs channel. + export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix From 5c34d665386f4053d666b0899ecca0639e500fbd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 15:38:24 +0200 Subject: [PATCH 0215/2548] Make flakes work with 'nix build --store ...' It was getting confused between logical and real store paths. Also, make fetchGit and fetchMercurial update allowedPaths properly. (Maybe the evaluator, rather than the caller of the evaluator, should apply toRealPath(), but that's a bigger change.) --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libexpr/primops/flake.cc | 13 +++++++------ src/libstore/download.cc | 2 ++ src/libstore/download.hh | 3 +++ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e79eacafe..00bbeb6d8 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -259,7 +259,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index cfe1bd871..596047ce3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -214,7 +214,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 8dd9599bd..c73487585 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -248,7 +248,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool FlakeRef ref(resolvedRef.baseRef()); ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); SourceInfo info(ref); - info.storePath = result.path; + info.storePath = result.storePath; return info; } @@ -294,21 +294,22 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) - state.allowedPaths->insert(sourceInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); // Guard against symlink attacks. Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); - if (!isInDir(flakeFile, sourceInfo.storePath)) - throw Error("flake file '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + Path realFlakeFile = state.store->toRealPath(flakeFile); + if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); Flake flake(flakeRef, sourceInfo); flake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash; - if (!pathExists(flakeFile)) + if (!pathExists(realFlakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); Value vInfo; - state.evalFile(flakeFile, vInfo); // FIXME: symlink attack + state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack state.forceAttrs(vInfo); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cb77cdc77..975cfd97d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -804,6 +804,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; + result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); return result; } @@ -912,6 +913,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url, expectedHash.to_string(), gotHash.to_string()); } + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 8acfe4e1a..aa8c34be2 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -43,6 +43,9 @@ struct DownloadResult struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; Path path; std::optional<std::string> etag; }; From 66b8a62101cb1dfe2e368346cf99efd32e9328ae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 17:33:56 +0200 Subject: [PATCH 0216/2548] nix: Add --print-build-logs flag This causes 'nix' to print build log output to stderr rather than showing the last log line in the progress bar. Log lines are prefixed by the name of the derivation (minus the version string), e.g. binutils> make[1]: Leaving directory '/build/binutils-2.31.1' binutils-wrapper> unpacking sources binutils-wrapper> patching sources ... binutils-wrapper> Using dynamic linker: '/nix/store/kr51dlsj9v5cr4n8700jliyz8v5b2q7q-bootstrap-stage0-glibc/lib/ld-linux-x86-64.so.2' bootstrap-stage2-gcc-wrapper> unpacking sources ... linux-headers> unpacking sources linux-headers> unpacking source archive /nix/store/8javli69jhj3bkql2c35gsj5vl91p382-linux-4.19.16.tar.xz --- src/libutil/util.hh | 1 + src/nix/main.cc | 10 ++++++++-- src/nix/progress-bar.cc | 39 +++++++++++++++++++++++++++++---------- src/nix/progress-bar.hh | 2 +- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f239bff3..23360de2e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -401,6 +401,7 @@ void ignoreException(); /* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" #define ANSI_RED "\e[31;1m" #define ANSI_GREEN "\e[32;1m" #define ANSI_BLUE "\e[34;1m" diff --git a/src/nix/main.cc b/src/nix/main.cc index 64c1dc357..4f87ad72b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -20,6 +20,8 @@ std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { mkFlag() @@ -41,6 +43,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs throw Exit(); }); + mkFlag() + .longName("print-build-logs") + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + mkFlag() .longName("version") .description("show version information") @@ -98,8 +105,7 @@ void mainWrapped(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) - startProgressBar(); + startProgressBar(args.printBuildLogs); args.command->prepare(); args.command->run(); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3..304f918cc 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "sync.hh" #include "store-api.hh" +#include "names.hh" #include <atomic> #include <map> @@ -38,6 +39,7 @@ private: std::map<ActivityType, uint64_t> expectedByType; bool visible = true; ActivityId parent; + std::optional<std::string> name; }; struct ActivitiesByType @@ -68,10 +70,16 @@ private: std::condition_variable quitCV, updateCV; + bool printBuildLogs; + bool isTTY; + public: - ProgressBar() + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) { + state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { @@ -109,8 +117,14 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - draw(state); + if (state.active) { + writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -141,6 +155,7 @@ public: auto nrRounds = getI(fields, 3); if (nrRounds != 1) i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; } if (type == actSubstitute) { @@ -217,11 +232,15 @@ public: auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(); + if (printBuildLogs) { + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(); + } } } @@ -395,9 +414,9 @@ public: } }; -void startProgressBar() +void startProgressBar(bool printBuildLogs) { - logger = new ProgressBar(); + logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO)); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index af8eda5a8..4d61175c2 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -4,7 +4,7 @@ namespace nix { -void startProgressBar(); +void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); From b0fc5bcee9f74c717d8ca564c193a5ad7846e5c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 15 May 2019 20:58:47 +0200 Subject: [PATCH 0217/2548] Start of flake documentation Imported from https://gist.github.com/edolstra/40da6e3a4d4ee8fd019395365e0772e7. The goal is to turn this into an RFC eventually and later to integrate it into the manual. --- doc/flakes/design.md | 566 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 566 insertions(+) create mode 100644 doc/flakes/design.md diff --git a/doc/flakes/design.md b/doc/flakes/design.md new file mode 100644 index 000000000..ebc229b09 --- /dev/null +++ b/doc/flakes/design.md @@ -0,0 +1,566 @@ +# Nix Flake MVP + +## Goals + +* To provide Nix repositories with an easy and standard way to + reference other Nix repositories. + +* To allow such references to be queried and updated automatically. + +* To provide a replacement for `nix-channel`, `NIX_PATH` and Hydra + jobset definitions. + +* To enable reproducible, hermetic evaluation of packages and NixOS + configurations. + +Things that we probably won't do in the initial iteration: + +* Sophisticated flake versioning, such as the ability to specify + version ranges on dependencies. + +* A way to specify the types of values provided by a flake. For the + most part, flakes can provide arbitrary Nix values, but there will + be some standard attribute names (e.g. `packages` must be a set of + installable derivations). + + +## Overview + +* A flake is (usually) a Git repository that contains a file named + `flake.nix` at top-level. + +* Flakes *provide* an attribute set of values, such as packages, + Nixpkgs overlays, NixOS modules, library functions, Hydra jobs, + `nix-shell` definitions, etc. + +* Flakes can *depend* on other flakes. + +* Flakes are referred to using a *flake reference*, which is either a + URL specifying its repository's location + (e.g. `github:NixOS/nixpkgs/release-18.09`) or an identifier + (e.g. `nixpkgs`) looked up in a *lock file* or *flake + registry*. They can also specify revisions, + e.g. `github:NixOS/nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f`. + +* The *flake registry* is a centrally maintained mapping (on + `nixos.org`) from flake identifiers to flake locations + (e.g. `nixpkgs -> github:NixOS/nixpkgs/release-18.09`). + +* A flake can contain a *lock file* (`flake.lock`) used when resolving + the dependencies in `flake.nix`. It maps flake references to + references containing revisions (e.g. `nixpkgs -> + github:NixOS/nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f`). + +* The `nix` command uses the flake registry as its default + installation source. For example, `nix build nixpkgs.hello` builds the + `hello` package provided by the `nixpkgs` flake listed in the + registry. `nix` will automatically download/upload the registry and + flakes as needed. + +* `nix build` without arguments will build the flake in the current + directory (or some parent). + +* The command `nix flake update` generates/updates `flake.lock` from + `flake.nix`. This should probably also be done automatically when + building from a local flake. + +* `nixos-rebuild` will build a configuration from a (locked) + flake. Evaluation will be done in pure mode to ensure there are no + unaccounted inputs. Thus the NixOS configuration can be reproduced + unambiguously from the top-level flake. + +* Nix code can query flake metadata such as `commitHash` (the Git + revision) or `date` (the date of the last commit). This is useful + for NixOS to compute the NixOS version string (which will be the + revision of the top-level configuration flake, uniquely identifying + the configuration). + +* Hydra jobset configurations will consist of a single flake + reference. Thus we can get rid of jobset inputs; any other needed + repositories can be fetched by the top-level flake. The top-level + flake can be locked or unlocked; if some dependencies are unlocked, + then Nix will fetch the latest revision for each. + + +## Example flake + +A flake is a Git repository that contains a file named +`flake.nix`. For example, here is the `flake.nix` for `dwarffs`, a +small repository that provides a single package and a single NixOS +module. + +```nix +{ + # The flake identifier. + name = "dwarffs"; + + # The epoch may be used in the future to determine how Nix + # expressions inside this flake are to be parsed. + epoch = 2018; + + # Some other metadata. + description = "A filesystem that fetches DWARF debug info from the Internet on demand"; + + # A list of flake references denoting the flakes that this flake + # depends on. Nix will resolve and fetch these flakes and pass them + # as a function argument to `provides` below. + # + # `flake:nixpkgs` denotes a flake named `nixpkgs` which is looked up + # in the flake registry, or in `flake.lock` inside this flake, if it + # exists. + requires = [ flake:nixpkgs ]; + + # The stuff provided by this flake. Flakes can provide whatever they + # want (convention over configuration), but some attributes have + # special meaning to tools / other flakes: for example, `packages` + # is used by the `nix` CLI to search for packages, and + # `nixosModules` is used by NixOS to automatically pull in the + # modules provided by a flake. + # + # `provides` takes a single argument named `deps` that contains + # the resolved set of flakes. (See below.) + provides = deps: { + + # This is searched by `nix`, so something like `nix install + # dwarffs.dwarffs` resolves to this `packages.dwarffs`. + packages.dwarffs = + with deps.nixpkgs.packages; + with deps.nixpkgs.builders; + with deps.nixpkgs.lib; + + stdenv.mkDerivation { + name = "dwarffs-0.1"; + + buildInputs = [ fuse nix nlohmann_json boost ]; + + NIX_CFLAGS_COMPILE = "-I ${nix.dev}/include/nix -include ${nix.dev}/include/nix/config.h -D_FILE_OFFSET_BITS=64"; + + src = cleanSource ./.; + + installPhase = + '' + mkdir -p $out/bin $out/lib/systemd/system + + cp dwarffs $out/bin/ + ln -s dwarffs $out/bin/mount.fuse.dwarffs + + cp ${./run-dwarffs.mount} $out/lib/systemd/system/run-dwarffs.mount + cp ${./run-dwarffs.automount} $out/lib/systemd/system/run-dwarffs.automount + ''; + }; + + # NixOS modules. + nixosModules.dwarffs = import ./module.nix deps; + + # Provide a single Hydra job (`hydraJobs.dwarffs`). + hydraJobs = deps.this.packages; + }; +} +``` + +Similarly, a minimal `flake.nix` for Nixpkgs: + +```nix +{ + name = "nixpkgs"; + + epoch = 2018; + + description = "A collection of packages for the Nix package manager"; + + provides = deps: + let pkgs = import ./. {}; in + { + lib = import ./lib; + + builders = { + inherit (pkgs) stdenv fetchurl; + }; + + packages = { + inherit (pkgs) hello nix fuse nlohmann_json boost; + }; + }; +} +``` +Note that `packages` is an unpolluted set of packages: non-package +values like `lib` or `fetchurl` are not part of it. + + +## Flake identifiers + +A flake has an identifier (e.g. `nixpkgs` or `dwarffs`). + + +## Flake references + +Flake references are a URI-like syntax to specify the physical +location of a flake (e.g. a Git repository) or to denote a lookup in +the flake registry or lock file. + +* `(flake:)?<flake-id>(/rev-or-ref(/rev)?)?` + + Look up a flake by ID in the flake lock file or in the flake + registry. These must specify an actual location for the flake using + the formats listed below. Note that in pure evaluation mode, the + flake registry is empty. + + Optionally, the `rev` or `ref` from the dereferenced flake can be + overriden. For example, + + > nixpkgs/19.09 + + uses the `19.09` branch of the `nixpkgs` flake's GitHub repository, + while + + > nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + uses the specified revision. For Git (rather than GitHub) + repositories, both the rev and ref must be given, e.g. + + > nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + +* `github:<owner>/<repo>(/<rev-or-ref>)?` + + A repository on GitHub. These differ from Git references in that + they're downloaded in a efficient way (via the tarball mechanism) + and that they support downloading a specific revision without + specifying a branch. `rev-or-ref` is either a commit hash (`rev`) + or a branch or tag name (`ref`). The default is `master` if none is + specified. Note that in pure evaluation mode, a commit hash must be + used. + + Flakes fetched in this manner expose `rev` and `date` attributes, + but not `revCount`. + + Examples: + + > github:edolstra/dwarffs + + > github:edolstra/dwarffs/unstable + + > github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 + +* > https://<server>/<path>.git(\?attr(&attr)*)? + + > ssh://<server>/<path>.git(\?attr(&attr)*)? + + > git://<server>/<path>.git(\?attr(&attr)*)? + + > file:///<path>(\?attr(&attr)*)? + + where `attr` is one of `rev=<rev>` or `ref=<ref>`. + + A Git repository fetched through https. Note that the path must end + in `.git`. The default for `ref` is `master`. + + Examples: + + > https://example.org/my/repo.git + > https://example.org/my/repo.git?ref=release-1.2.3 + > https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + +* > /path.git(\?attr(&attr)*)? + + Like `file://path.git`, but if no `ref` or `rev` is specified, the + (possibly dirty) working tree will be used. Using a working tree is + not allowed in pure evaluation mode. + + Examples: + + > /path/to/my/repo + + > /path/to/my/repo?ref=develop + + > /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + +* > https://<server>/<path>.tar.xz(?hash=<sri-hash>) + + > file:///<path>.tar.xz(?hash=<sri-hash>) + + A flake distributed as a tarball. In pure evaluation mode, an SRI + hash is mandatory. It exposes a `date` attribute, being the newest + file inside the tarball. + + Example: + + > https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + + > https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c + +Note: currently, there can be only one flake per Git repository, and +it must be at top-level. In the future, we may want to add a field +(e.g. `dir=<dir>`) to specify a subdirectory inside the repository. + + +## Flake lock files + +This is a JSON file named `flake.lock` that maps flake identifiers +used in the corresponding `flake.nix` to "immutable" flake references; +that is, flake references that contain a revision (for Git +repositories) or a content hash (for tarballs). + +Example: + +```json +{ + "nixpkgs": "github:NixOS/nixpkgs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553", + "foo": "https://example.org/foo.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c" +} +``` + + +## `provides` + +The flake attribute `provides` is a function that takes an argument +named `deps` and returns a (mostly) arbitrary attrset of values. Some +of the standard result attributes: + +* `packages`: A set of installable derivations used by the `nix` + command. That is, commands such as `nix install` ignore all other + flake attributes. + +* `hydraJobs`: Used by Hydra. + +* `nixosModules`: An attrset of NixOS modules. + +* `nixosSystems`: An attrset of calls to `evalModules`, i.e. things + that `nixos-rebuild` can switch to. (Maybe this is superfluous, but + we need to avoid a situation where `nixos-rebuild` needs to fetch + its own `nixpkgs` just to do `evalModules`.) + +* `shell`: A specification of a development environment in some TBD + format. + +The function argument `flakes` is an attrset that contains an +attribute for each dependency specified in `requires`. (Should it +contain transitive dependencies? Probably not.) Each attribute is an +attrset containing the `provides` of the dependency, in addition to +the following attributes: + +* `path`: The path to the flake's source code. Useful when you want to + use non-Nix artifacts from the flake, or if you want to *store* the + source code of the dependency in a derivation. (For example, we + could store the sources of all flake dependencies in a NixOS system + configuration, as a generalization of + `system.copySystemConfiguration`.) + +* `meta`: An attrset containing the following: + + * `description` + + * `commitHash` (or `rev`?) (not for tarball flakes): The Git commit + hash. + + * `date`: The timestamp of the most recent commit (for Git + repositories), or the timestamp of the most recently modified file + (for tarballs). + + * `revCount` (for Git flakes, but not GitHub flakes): The number of + ancestors of the revision. Useful for generating version strings. + + +## Non-flake dependencies + +It may be useful to pull in repositories that are not flakes +(i.e. don't contain a `flake.nix`). This could be done in two ways: + +* Allow flakes not to have a `flake.nix` file, in which case it's a + flake with no requires and no provides. The downside of this + approach is that we can't detect accidental use of a non-flake + repository. (Also, we need to conjure up an identifier somehow.) + +* Add a flake attribute to specifiy non-flake dependencies, e.g. + + > nonFlakeRequires.foobar = github:foo/bar; + + +## Flake registry + +The flake registry maps flake IDs to flake references (where the +latter cannot be another indirection, i.e. it must not be a +`flake:<flake-id>` reference). + +The default registry is kept at +`https://nixos.org/flake-registry.json`. It looks like this: + +```json +{ + "version": 1, + "flakes": { + "dwarffs": { + "uri": "github:edolstra/dwarffs/flake" + }, + "nixpkgs": { + "uri": "github:NixOS/nixpkgs/release-18.09" + } + } +} +``` + +Nix automatically (re)downloads the registry. The downloaded file is a +GC root so the registry remains available if nixos.org is unreachable. +TBD: when to redownload? + + +## Nix UI + +Commands for registry / user flake configuration: + +* `nix flake list`: Show all flakes in the registry. + +* `nix flake add <flake-ref>`: Add or override a flake to/in the + user's flake configuration (`~/.config/nix/flakes.nix`). For + example, `nix flake add nixpkgs/nixos-18.03` overrides the `nixpkgs` + flake to use the `nixos-18.03` branch. There should also be a way to + add multiple branches/revisions of the same flake by giving them a + different ID, e.g. `nix flake add --id nixpkgs-ancient + nixpkgs/nixos-16.03`). + +* `nix flake remove <flake-id>`: Remove a flake from the user's flake + configuration. Any flake with the same ID in the registry remains + available. + +* `nix flake lock <flake-id>`: Lock a flake. For example, `nix flake + lock nixpkgs` pins `nixpkgs` to the current revision. + +Commands for creating/modifying a flake: + +* `nix flake init`: Create a `flake.nix` in the current directory. + +* `nix flake update`: Update the lock file for the `flake.nix` in the + current directory. In most cases, this should be done + automatically. (E.g. `nix build` should automatically update the + lock file is a new dependency is added to `flake.nix`.) + +* `nix flake check`: Do some checks on the flake, e.g. check that all + `packages` are really packages. + +* `nix flake clone`: Do a Git clone of the flake repository. This is a + convenience to easily start hacking on a flake. E.g. `nix flake + clone dwarffs` clones the `dwarffs` GitHub repository to `./dwarffs`. + +TODO: maybe the first set of commands should have a different name +from the second set. + +Flags / configuration options: + +* `--flakes (<flake-id>=<flake-ref>)*`: add/override some flakes. + +* (In `nix`) `--flake <flake-ref>`: set the specified flake as the + installation source. E.g. `nix build --flake ./my-nixpkgs hello`. + +The default installation source in `nix` is the `packages` from all +flakes in the registry, that is: +``` +builtins.mapAttrs (flakeName: flakeInfo: + (getFlake flakeInfo.uri).${flakeName}.provides.packages or {}) + builtins.flakeRegistry +``` +(where `builtins.flakeRegistry` is the global registry with user +overrides applied, and `builtins.getFlake` downloads a flake and +resolves its dependencies.) + +It may be nice to extend the default installation source with the +`packages` from the flake in the current directory, so that + +> nix build hello + +does something similar to the old + +> nix-build -A hello + +Specifically, it builds `packages.hello` from the flake in the current +directory. Of course, this creates some ambiguity if there is a flake +in the registry named `hello`. + +Maybe the command + +> nix shell + +should do something like use `provides.shell` to initialize the shell, +but probably we should ditch `nix shell` / `nix-shell` for direnv. + + +## Pure evaluation and caching + +Flake evaluation should be done in pure mode. Thus: + +* Flakes cannot do `NIX_PATH` lookups via the `<...>` syntax. + +* They can't read random stuff from non-flake directories, such as + `~/.nix/config.nix` or overlays. + +This enables aggressive caching or precomputation of Nixpkgs package +sets. For example, for a particular Nixpkgs flake closure (as +identified by, say, a hash of the fully-qualified flake references +after dependency resolution) and system type, an attribute like +`packages.hello` should always evaluate to the same derivation. So we +can: + +* Keep a local evaluation cache (say `~/.cache/nix/eval.sqlite`) + mapping `(<flake-closure-hash, <attribute>) -> (<drv-name>, + <drv-output-paths>, <whatever other info we want to cache>)`. + +* Download a precomputed cache + (e.g. `https://releases.nixos.org/eval/<flake-closure-hash>.sqlite`). So + a command like `nix search` could avoid evaluating Nixpkgs entirely. + +Of course, this doesn't allow overlays. With pure evaluation, the only +way to have these is to define a top-level flake that depends on the +Nixpkgs flake and somehow passes in a set of overlays. + +TODO: in pure mode we have to pass the system type explicitly! + + +## Hydra jobset dependencies + +Hydra can use the flake dependency resolution mechanism to fetch +dependencies. This allows us to get rid of jobset configuration in the +web interface: a jobset only requires a flake reference. That is, *a +jobset is a flake*. Hydra then just builds the `hydraJobs` attrset +`provide`d by the flake. (It omitted, maybe it can build `packages`.) + + +## NixOS system configuration + +NixOS currently contains a lot of modules that really should be moved +into their own repositories. For example, it contains a Hydra module +that duplicates the one in the Hydra repository. Also, we want +reproducible evaluation for NixOS system configurations. So NixOS +system configurations should be stored as flakes in (local) Git +repositories. + +`my-system/flake.nix`: + +```nix +{ + provides = flakes: { + nixosSystems.default = + flakes.nixpkgs.lib.evalModules { + modules = + [ { networking.firewall.enable = true; + hydra.useSubstitutes = true; + } + # The latter could be extracted automatically from `flakes`. + flakes.dwarffs.nixosModules.dwarffs + flakes.hydra.nixosModules.hydra + ]; + }; + }; + + requires = + [ "nixpkgs/nixos-18.09" + "dwarffs" + "hydra" + ... lots of other module flakes ... + ]; +} +``` + +We can then build the system: +``` +nixos-rebuild switch --flake ~/my-system +``` +This performs dependency resolution starting at `~/my-system/flake.nix` +and builds the `system` attribute in `nixosSystems.default`. From 7c20ee448fa924d898bcebf84bd0a7caf368a656 Mon Sep 17 00:00:00 2001 From: Matthew Bauer <mjbauer95@gmail.com> Date: Wed, 15 May 2019 22:04:39 -0400 Subject: [PATCH 0218/2548] Sync NIX_PROFILES between single-user and multi-user modes When we are in single user mode, we still want to have access to profiles. This way things in Nixpkgs that rely on them getting set accurately are done in both cases. The point where I hit this is with using aspell which looks in NIX_PROFILES: https://github.com/NixOS/nixpkgs/blob/master/pkgs/development/libraries/aspell/default.nix Before this patch, NIX_PROFILES was never set in single user mode! This corrects that. --- scripts/nix-profile.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index f3cfa157c..85f1d6e5d 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -57,7 +57,7 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -79,5 +79,5 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES + unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR fi From 92f461e4f46b595e3712b41c30b8403905dd8673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer <mjbauer95@gmail.com> Date: Wed, 15 May 2019 22:24:24 -0400 Subject: [PATCH 0219/2548] =?UTF-8?q?Don=E2=80=99t=20set=20NIX=5FREMOTE=3D?= =?UTF-8?q?daemon=20in=20daemon=20profile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is now autodetected. There is no need to put it in the profile. --- scripts/nix-profile-daemon.sh.in | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 6940969cc..23da5e855 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -2,12 +2,6 @@ if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -# Set up secure multi-user builds: non-root users build through the -# Nix daemon. -if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then - export NIX_REMOTE=daemon -fi - export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" From d9ad3723d59d9df2fb3c89335b5d9239f1860ec9 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Wed, 1 May 2019 11:38:48 +0200 Subject: [PATCH 0220/2548] Fixed issue 65 lockfile updating --- src/libexpr/primops/flake.cc | 91 +++++++++++++++++++++--------------- src/libexpr/primops/flake.hh | 6 +-- src/nix/command.hh | 4 +- src/nix/flake.cc | 9 ++-- src/nix/installables.cc | 16 +++---- tests/flakes.sh | 39 ++++++++-------- 6 files changed, 90 insertions(+), 75 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c73487585..88eadff55 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -289,8 +289,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe FlakeRef resolvedRef = sourceInfo.resolvedRef; - resolvedRef = sourceInfo.resolvedRef; // `resolvedRef` is now immutable - state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) @@ -368,35 +366,59 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al return nonFlake; } -/* Given a flake reference, recursively fetch it and its - dependencies. - FIXME: this should return a graph of flakes. -*/ -ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, - RegistryAccess registryAccess, bool isTopFlake) +LockFile entryToLockFile(const LockFile::FlakeEntry & entry) +{ + LockFile lockFile; + lockFile.flakeEntries = entry.flakeEntries; + lockFile.nonFlakeEntries = entry.nonFlakeEntries; + return lockFile; +} + +ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, + LockFile lockFile, bool isTopFlake = false) { bool allowRegistries = registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake); - Flake flake = getFlake(state, topRef, allowRegistries); - LockFile lockFile; - - if (isTopFlake) - lockFile = readLockFile(flake.storePath + flake.resolvedRef.subdir + "/flake.lock"); // FIXME: symlink attack + Flake flake = getFlake(state, flakeRef, allowRegistries); ResolvedFlake deps(flake); - for (auto & nonFlakeInfo : flake.nonFlakeRequires) - deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); + for (auto & nonFlakeInfo : flake.nonFlakeRequires) { + FlakeRef ref = nonFlakeInfo.second; + auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); + if (i != lockFile.nonFlakeEntries.end()) ref = i->second.ref; + deps.nonFlakeDeps.push_back(getNonFlake(state, ref, nonFlakeInfo.first)); + } for (auto newFlakeRef : flake.requires) { + FlakeRef ref = newFlakeRef; + LockFile newLockFile; auto i = lockFile.flakeEntries.find(newFlakeRef); - if (i != lockFile.flakeEntries.end()) newFlakeRef = i->second.ref; - // FIXME: propagate lockFile downwards - deps.flakeDeps.push_back(resolveFlake(state, newFlakeRef, registryAccess, false)); + if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible + ref = i->second.ref; + newLockFile = entryToLockFile(i->second); + } + deps.flakeDeps.push_back(resolveFlakeFromLockFile(state, ref, registryAccess, newLockFile)); } return deps; } +/* Given a flake reference, recursively fetch it and its dependencies. + FIXME: this should return a graph of flakes. +*/ +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, RegistryAccess registryAccess, + bool recreateLockFile) +{ + bool allowRegistries = registryAccess == AllowRegistry || registryAccess == AllowRegistryAtTop; + Flake flake = getFlake(state, topRef, allowRegistries); + LockFile lockFile; + + if (!recreateLockFile) // If recreateLockFile, start with an empty lockfile + lockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack + + return resolveFlakeFromLockFile(state, topRef, registryAccess, lockFile, true); +} + LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash); @@ -410,31 +432,25 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlak return entry; } -static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef) +static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef, bool recreateLockFile) { - ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, AllowRegistry); - LockFile::FlakeEntry entry = dependenciesToFlakeEntry(resFlake); - LockFile lockFile; - lockFile.flakeEntries = entry.flakeEntries; - lockFile.nonFlakeEntries = entry.nonFlakeEntries; - return lockFile; + ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, AllowRegistry, recreateLockFile); + return entryToLockFile(dependenciesToFlakeEntry(resFlake)); } -void updateLockFile(EvalState & state, const FlakeUri & flakeUri) +void updateLockFile(EvalState & state, const FlakeUri & uri, bool recreateLockFile) { - // FIXME: We are writing the lockfile to the store here! Very bad practice! - FlakeRef flakeRef = FlakeRef(flakeUri); + FlakeRef flakeRef = FlakeRef(uri); + auto lockFile = makeLockFile(state, flakeRef, recreateLockFile); if (auto refData = std::get_if<FlakeRef::IsPath>(&flakeRef.data)) { - auto lockFile = makeLockFile(state, flakeRef); - writeLockFile(lockFile, refData->path + "/" + flakeRef.subdir + "/flake.lock"); + writeLockFile(lockFile, refData->path + (flakeRef.subdir == "" ? "" : "/" + flakeRef.subdir) + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git. Otherwise, // exportGit will fail to copy it to the Nix store. - runProgram("git", true, - { "-C", refData->path, "add", "--intent-to-add", - (flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" }); + runProgram("git", true, { "-C", refData->path, "add", + (flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" }); } else - throw Error("flakeUri %s can't be updated because it is not a path", flakeUri); + throw Error("flakeUri %s can't be updated because it is not a path", uri); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) @@ -485,16 +501,17 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v) +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v, bool recreateLockFile) { - callFlake(state, resolveFlake(state, flakeRef, registryAccess), v); + callFlake(state, resolveFlake(state, flakeRef, registryAccess, recreateLockFile), v); } // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v); + evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v, false); + // `recreateLockFile == false` because this is the evaluation stage, which should be pure, and hence not recreate lockfiles. } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 691f68b0a..e3481e99e 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -45,7 +45,7 @@ Path getUserRegistryPath(); enum RegistryAccess { DisallowRegistry, AllowRegistry, AllowRegistryAtTop }; -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v); +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v, bool recreateLockFile); std::shared_ptr<FlakeRegistry> readRegistry(const Path &); @@ -103,9 +103,9 @@ struct ResolvedFlake ResolvedFlake(const Flake & flake) : flake(flake) {} }; -ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess registryAccess, bool isTopFlake = true); +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess, bool recreateLockFile); -void updateLockFile(EvalState &, const FlakeUri &); +void updateLockFile(EvalState &, const FlakeUri &, bool recreateLockFile); void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path); } diff --git a/src/nix/command.hh b/src/nix/command.hh index 640c6cd16..32a5047a8 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -76,10 +76,10 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { std::optional<Path> file; - bool updateLockFile = true; - SourceExprCommand(); + bool recreateLockFile = false; + ref<EvalState> getEvalState(); std::vector<std::shared_ptr<Installable>> parseInstallables( diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0af368570..d2cdf6fc9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -114,7 +114,8 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop); + bool recreateLockFile = false; + ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop, recreateLockFile); std::queue<ResolvedFlake> todo; todo.push(resFlake); @@ -132,7 +133,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs } }; -struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs +struct CmdFlakeUpdate : StoreCommand, FlakeCommand, MixEvalArgs { std::string name() override { @@ -148,8 +149,8 @@ struct CmdFlakeUpdate : StoreCommand, GitRepoCommand, MixEvalArgs { auto evalState = std::make_shared<EvalState>(searchPath, store); - if (gitPath == "") gitPath = absPath("."); - updateLockFile(*evalState, gitPath); + bool recreateLockFile = true; + updateLockFile(*evalState, flakeUri, recreateLockFile); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index db67952e1..6d784002a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -23,9 +23,9 @@ SourceExprCommand::SourceExprCommand() .dest(&file); mkFlag() - .longName("no-update") - .description("don't create/update flake lock files") - .set(&updateLockFile, false); + .longName("recreate-lock-file") + .description("recreate lock file from scratch") + .set(&recreateLockFile, true); } ref<EvalState> SourceExprCommand::getEvalState() @@ -157,13 +157,11 @@ struct InstallableFlake : InstallableValue Value * toValue(EvalState & state) override { - auto path = std::get_if<FlakeRef::IsPath>(&flakeRef.data); - if (cmd.updateLockFile && path) { - updateLockFile(state, path->path); - } - auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake); + if (std::get_if<FlakeRef::IsPath>(&flakeRef.data)) + updateLockFile(state, flakeRef.to_string(), cmd.recreateLockFile); + + makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake, cmd.recreateLockFile); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; diff --git a/tests/flakes.sh b/tests/flakes.sh index 40ba42715..8b68aea65 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -9,11 +9,11 @@ clearStore registry=$TEST_ROOT/registry.json -flake1=$TEST_ROOT/flake1 -flake2=$TEST_ROOT/flake2 -flake3=$TEST_ROOT/flake3 +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 +flake3Dir=$TEST_ROOT/flake3 -for repo in $flake1 $flake2 $flake3; do +for repo in $flake1Dir $flake2Dir $flake3Dir; do rm -rf $repo mkdir $repo git -C $repo init @@ -21,7 +21,7 @@ for repo in $flake1 $flake2 $flake3; do git -C $repo config user.name "Foobar" done -cat > $flake1/flake.nix <<EOF +cat > $flake1Dir/flake.nix <<EOF { name = "flake1"; @@ -36,11 +36,11 @@ cat > $flake1/flake.nix <<EOF } EOF -cp ./simple.nix ./simple.builder.sh ./config.nix $flake1/ -git -C $flake1 add flake.nix simple.nix simple.builder.sh config.nix -git -C $flake1 commit -m 'Initial' +cp ./simple.nix ./simple.builder.sh ./config.nix $flake1Dir/ +git -C $flake1Dir add flake.nix simple.nix simple.builder.sh config.nix +git -C $flake1Dir commit -m 'Initial' -cat > $flake2/flake.nix <<EOF +cat > $flake2Dir/flake.nix <<EOF { name = "flake2"; @@ -56,8 +56,8 @@ cat > $flake2/flake.nix <<EOF } EOF -git -C $flake2 add flake.nix -git -C $flake2 commit -m 'Initial' +git -C $flake2Dir add flake.nix +git -C $flake2Dir commit -m 'Initial' cat > $flake3/flake.nix <<EOF { @@ -82,13 +82,13 @@ cat > $registry <<EOF { "flakes": { "flake1": { - "uri": "file://$flake1" + "uri": "file://$flake1Dir" }, "flake2": { - "uri": "file://$flake2" + "uri": "file://$flake2Dir" }, "flake3": { - "uri": "file://$flake3" + "uri": "file://$flake3Dir" }, "nixpkgs": { "uri": "flake1" @@ -121,13 +121,12 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1: (! nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar) # But should succeed in impure mode. -# FIXME: this currently fails. -#nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure +nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure # Test automatic lock file generation. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar -[[ -e $flake2/flake.lock ]] -git -C $flake2 commit flake.lock -m 'Add flake.lock' +nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar +[[ -e $flake2Dir/flake.lock ]] +git -C $flake2Dir commit flake.lock -m 'Add flake.lock' # Rerunning the build should not change the lockfile. #nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar @@ -137,7 +136,7 @@ git -C $flake2 commit flake.lock -m 'Add flake.lock' nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar # Or without a registry. -nix build -o $TEST_ROOT/result file://$flake2:bar +nix build -o $TEST_ROOT/result file://$flake2Dir:bar # Test whether indirect dependencies work. #nix build -o $TEST_ROOT/result --flake-registry $registry $flake3:xyzzy From 98f20dee41e9d4dccb5a6bbbd956ab856c5f7929 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Wed, 1 May 2019 11:38:48 +0200 Subject: [PATCH 0221/2548] Give errors in resolveFlake If DontUpdate but the lockfile isn't correct --- src/libexpr/primops/flake.cc | 70 ++++++++++++++++++++---------------- src/libexpr/primops/flake.hh | 8 ++--- src/nix/flake.cc | 3 +- src/nix/installables.cc | 2 +- tests/config.nix | 20 +++++++++++ tests/flakes.sh | 8 ++--- 6 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 tests/config.nix diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 88eadff55..c576a8b3e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -50,8 +50,7 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) if (!flakeRef.isImmutable()) throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); - Hash hash = Hash((std::string) json["contentHash"]); - LockFile::FlakeEntry entry(flakeRef, hash); + LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"])); auto nonFlakeRequires = json["nonFlakeRequires"]; @@ -59,9 +58,8 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - Hash hash = Hash((std::string) i->value("contentHash", "")); - LockFile::NonFlakeEntry newEntry(flakeRef, hash); - entry.nonFlakeEntries.insert_or_assign(i.key(), newEntry); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); + entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } auto requires = json["requires"]; @@ -89,10 +87,10 @@ LockFile readLockFile(const Path & path) for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); - LockFile::NonFlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"])); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), entry); + throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); + lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } auto requires = json["requires"]; @@ -374,19 +372,25 @@ LockFile entryToLockFile(const LockFile::FlakeEntry & entry) return lockFile; } -ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, - LockFile lockFile, bool isTopFlake = false) +ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, + ShouldUpdateLockFile update, LockFile lockFile = {}) { - bool allowRegistries = registryAccess == AllowRegistry || (registryAccess == AllowRegistryAtTop && isTopFlake); - Flake flake = getFlake(state, flakeRef, allowRegistries); + Flake flake = getFlake(state, flakeRef, update != DontUpdate); ResolvedFlake deps(flake); for (auto & nonFlakeInfo : flake.nonFlakeRequires) { FlakeRef ref = nonFlakeInfo.second; auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); - if (i != lockFile.nonFlakeEntries.end()) ref = i->second.ref; - deps.nonFlakeDeps.push_back(getNonFlake(state, ref, nonFlakeInfo.first)); + if (i != lockFile.nonFlakeEntries.end()) { + NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); + if (nonFlake.hash != i->second.contentHash) + throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); + deps.nonFlakeDeps.push_back(nonFlake); + } else { + if (update == DontUpdate) throw Error("the lockfile requires updating nonflake dependency %s in DontUpdate mode", nonFlakeInfo.first); + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); + } } for (auto newFlakeRef : flake.requires) { @@ -394,10 +398,14 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake LockFile newLockFile; auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible - ref = i->second.ref; - newLockFile = entryToLockFile(i->second); + ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, update, entryToLockFile(i->second)); + if (newResFlake.flake.hash != i->second.contentHash) + throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); + deps.flakeDeps.push_back(newResFlake); + } else { + if (update == DontUpdate) throw Error("the lockfile requires updating flake dependency %s in DontUpdate mode", newFlakeRef.to_string()); + deps.flakeDeps.push_back(resolveFlakeFromLockFile(state, newFlakeRef, update)); } - deps.flakeDeps.push_back(resolveFlakeFromLockFile(state, ref, registryAccess, newLockFile)); } return deps; @@ -406,17 +414,18 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake /* Given a flake reference, recursively fetch it and its dependencies. FIXME: this should return a graph of flakes. */ -ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, RegistryAccess registryAccess, - bool recreateLockFile) +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, ShouldUpdateLockFile update) { - bool allowRegistries = registryAccess == AllowRegistry || registryAccess == AllowRegistryAtTop; - Flake flake = getFlake(state, topRef, allowRegistries); + if (!std::get_if<FlakeRef::IsPath>(&topRef.data)) update = DontUpdate; + Flake flake = getFlake(state, topRef, update != DontUpdate); LockFile lockFile; - if (!recreateLockFile) // If recreateLockFile, start with an empty lockfile + if (update != RecreateLockFile) { + // If recreateLockFile, start with an empty lockfile lockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack + } - return resolveFlakeFromLockFile(state, topRef, registryAccess, lockFile, true); + return resolveFlakeFromLockFile(state, topRef, update, lockFile); } LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) @@ -426,15 +435,17 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlak for (auto & newResFlake : resolvedFlake.flakeDeps) entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake)); - for (auto & nonFlake : resolvedFlake.nonFlakeDeps) - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, LockFile::NonFlakeEntry(nonFlake.resolvedRef, nonFlake.hash)); + for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { + LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); + } return entry; } static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef, bool recreateLockFile) { - ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, AllowRegistry, recreateLockFile); + ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); return entryToLockFile(dependenciesToFlakeEntry(resFlake)); } @@ -501,17 +512,16 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v, bool recreateLockFile) +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, ShouldUpdateLockFile update, Value & v) { - callFlake(state, resolveFlake(state, flakeRef, registryAccess, recreateLockFile), v); + callFlake(state, resolveFlake(state, flakeRef, update), v); } // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? DisallowRegistry : AllowRegistryAtTop, v, false); - // `recreateLockFile == false` because this is the evaluation stage, which should be pure, and hence not recreate lockfiles. + evalSettings.pureEval ? DontUpdate : UpdateLockFile, v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index e3481e99e..132439b93 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -43,9 +43,9 @@ typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; Path getUserRegistryPath(); -enum RegistryAccess { DisallowRegistry, AllowRegistry, AllowRegistryAtTop }; +enum ShouldUpdateLockFile { DontUpdate, UpdateLockFile, RecreateLockFile}; -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, RegistryAccess registryAccess, Value & v, bool recreateLockFile); +void makeFlakeValue(EvalState &, const FlakeRef &, ShouldUpdateLockFile, Value &); std::shared_ptr<FlakeRegistry> readRegistry(const Path &); @@ -84,8 +84,8 @@ struct NonFlake FlakeRef originalRef; FlakeRef resolvedRef; std::optional<uint64_t> revCount; + Hash hash; Path storePath; - Hash hash; // content hash // date NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; @@ -103,7 +103,7 @@ struct ResolvedFlake ResolvedFlake(const Flake & flake) : flake(flake) {} }; -ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, RegistryAccess, bool recreateLockFile); +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, ShouldUpdateLockFile); void updateLockFile(EvalState &, const FlakeUri &, bool recreateLockFile); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d2cdf6fc9..fc0fc76b4 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -114,8 +114,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs FlakeRef flakeRef(flakeUri); - bool recreateLockFile = false; - ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, AllowRegistryAtTop, recreateLockFile); + ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, UpdateLockFile); std::queue<ResolvedFlake> todo; todo.push(resFlake); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 6d784002a..25f3f4f9d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -161,7 +161,7 @@ struct InstallableFlake : InstallableValue if (std::get_if<FlakeRef::IsPath>(&flakeRef.data)) updateLockFile(state, flakeRef.to_string(), cmd.recreateLockFile); - makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake, cmd.recreateLockFile); + makeFlakeValue(state, flakeRef, cmd.recreateLockFile ? RecreateLockFile : UpdateLockFile, *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; diff --git a/tests/config.nix b/tests/config.nix new file mode 100644 index 000000000..03810d57a --- /dev/null +++ b/tests/config.nix @@ -0,0 +1,20 @@ +with import <nix/config.nix>; + +rec { + inherit shell; + + path = coreutils; + + system = "x86_64-linux"; + + shared = builtins.getEnv "_NIX_TEST_SHARED"; + + mkDerivation = args: + derivation ({ + inherit system; + builder = shell; + args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + PATH = path; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; +} diff --git a/tests/flakes.sh b/tests/flakes.sh index 8b68aea65..d720eaf23 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -59,7 +59,7 @@ EOF git -C $flake2Dir add flake.nix git -C $flake2Dir commit -m 'Initial' -cat > $flake3/flake.nix <<EOF +cat > $flake3Dir/flake.nix <<EOF { name = "flake3"; @@ -75,8 +75,8 @@ cat > $flake3/flake.nix <<EOF } EOF -git -C $flake3 add flake.nix -git -C $flake3 commit -m 'Initial' +git -C $flake3Dir add flake.nix +git -C $flake3Dir commit -m 'Initial' cat > $registry <<EOF { @@ -139,4 +139,4 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar nix build -o $TEST_ROOT/result file://$flake2Dir:bar # Test whether indirect dependencies work. -#nix build -o $TEST_ROOT/result --flake-registry $registry $flake3:xyzzy +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy From c0559a1d6092b94c7ce46a065d72fc74e4db51dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <v@cunat.cz> Date: Fri, 17 May 2019 09:50:42 +0200 Subject: [PATCH 0222/2548] docs: describe $IN_NIX_SHELL values (#2796) See commit 1bffd83e1a9 --- doc/manual/command-ref/env-common.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index c532ffdde..6a3aaae71 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -14,7 +14,8 @@ <varlistentry><term><envar>IN_NIX_SHELL</envar></term> <listitem><para>Indicator that tells if the current environment was set up by - <command>nix-shell</command>.</para></listitem> + <command>nix-shell</command>. Since Nix 2.0 the values are + <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem> </varlistentry> From ef6ae61503bed7afa73a45ca6a4fb3597a9e514d Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Tue, 14 May 2019 11:34:45 +0200 Subject: [PATCH 0223/2548] Lockfile handling in `resolveFlake` is fixed --- src/libexpr/primops/flake.cc | 136 +++++++++++++++++++++-------------- src/libexpr/primops/flake.hh | 17 +++-- src/nix/command.hh | 4 ++ src/nix/flake.cc | 4 +- src/nix/installables.cc | 18 ++++- tests/config.nix | 20 ------ tests/flakes.sh | 35 ++++++++- 7 files changed, 149 insertions(+), 85 deletions(-) delete mode 100644 tests/config.nix diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c576a8b3e..3cbb0c1ef 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -372,10 +372,58 @@ LockFile entryToLockFile(const LockFile::FlakeEntry & entry) return lockFile; } -ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, - ShouldUpdateLockFile update, LockFile lockFile = {}) +LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - Flake flake = getFlake(state, flakeRef, update != DontUpdate); + LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash); + + for (auto & info : resolvedFlake.flakeDeps) + entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); + + for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { + LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash); + entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); + } + + return entry; +} + +bool allowedToWrite (HandleLockFile handle) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return false; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return false; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return false; + else assert(false); +} + +bool recreateLockFile (HandleLockFile handle) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return false; + else if (handle == UpdateLockFile) return false; + else if (handle == UseUpdatedLockFile) return false; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +bool allowedToUseRegistries (HandleLockFile handle, bool isTopRef) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return isTopRef; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return true; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, + HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false) +{ + Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); ResolvedFlake deps(flake); @@ -388,23 +436,23 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); deps.nonFlakeDeps.push_back(nonFlake); } else { - if (update == DontUpdate) throw Error("the lockfile requires updating nonflake dependency %s in DontUpdate mode", nonFlakeInfo.first); + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("the lockfile requires updating nonflake dependency %s in AllPure mode", nonFlakeInfo.first); deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); } } for (auto newFlakeRef : flake.requires) { - FlakeRef ref = newFlakeRef; - LockFile newLockFile; auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible - ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, update, entryToLockFile(i->second)); + ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); if (newResFlake.flake.hash != i->second.contentHash) throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); - deps.flakeDeps.push_back(newResFlake); + deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); } else { - if (update == DontUpdate) throw Error("the lockfile requires updating flake dependency %s in DontUpdate mode", newFlakeRef.to_string()); - deps.flakeDeps.push_back(resolveFlakeFromLockFile(state, newFlakeRef, update)); + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("the lockfile requires updating flake dependency %s in AllPure mode", newFlakeRef.to_string()); + deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); } } @@ -414,54 +462,37 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake /* Given a flake reference, recursively fetch it and its dependencies. FIXME: this should return a graph of flakes. */ -ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, ShouldUpdateLockFile update) +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { - if (!std::get_if<FlakeRef::IsPath>(&topRef.data)) update = DontUpdate; - Flake flake = getFlake(state, topRef, update != DontUpdate); + Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); LockFile lockFile; - if (update != RecreateLockFile) { + if (!recreateLockFile (handleLockFile)) { // If recreateLockFile, start with an empty lockfile lockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack } - return resolveFlakeFromLockFile(state, topRef, update, lockFile); + ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); + lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); + + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { + writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else std::cout << "Cannot write lockfile because the FlakeRef isn't of the form IsPath." << std::endl; + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + std::cout << "Using updating lockfile without writing it to file" << std::endl; + + return resFlake; } -LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) +void updateLockFile (EvalState & state, const FlakeUri & flakeUri, bool recreateLockFile) { - LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash); - - for (auto & newResFlake : resolvedFlake.flakeDeps) - entry.flakeEntries.insert_or_assign(newResFlake.flake.originalRef, dependenciesToFlakeEntry(newResFlake)); - - for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { - LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash); - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); - } - - return entry; -} - -static LockFile makeLockFile(EvalState & evalState, FlakeRef & flakeRef, bool recreateLockFile) -{ - ResolvedFlake resFlake = resolveFlake(evalState, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); - return entryToLockFile(dependenciesToFlakeEntry(resFlake)); -} - -void updateLockFile(EvalState & state, const FlakeUri & uri, bool recreateLockFile) -{ - FlakeRef flakeRef = FlakeRef(uri); - auto lockFile = makeLockFile(state, flakeRef, recreateLockFile); - if (auto refData = std::get_if<FlakeRef::IsPath>(&flakeRef.data)) { - writeLockFile(lockFile, refData->path + (flakeRef.subdir == "" ? "" : "/" + flakeRef.subdir) + "/flake.lock"); - - // Hack: Make sure that flake.lock is visible to Git. Otherwise, - // exportGit will fail to copy it to the Nix store. - runProgram("git", true, { "-C", refData->path, "add", - (flakeRef.subdir == "" ? "" : flakeRef.subdir + "/") + "flake.lock" }); - } else - throw Error("flakeUri %s can't be updated because it is not a path", uri); + FlakeRef flakeRef(flakeUri); + resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) @@ -471,7 +502,8 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); - for (const ResolvedFlake newResFlake : resFlake.flakeDeps) { + for (auto info : resFlake.flakeDeps) { + const ResolvedFlake newResFlake = info.second; auto vFlake = state.allocAttr(v, newResFlake.flake.id); callFlake(state, newResFlake, *vFlake); } @@ -512,16 +544,16 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) // Return the `provides` of the top flake, while assigning to `v` the provides // of the dependencies as well. -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, ShouldUpdateLockFile update, Value & v) +void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, HandleLockFile handle, Value & v) { - callFlake(state, resolveFlake(state, flakeRef, update), v); + callFlake(state, resolveFlake(state, flakeRef, handle), v); } // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? DontUpdate : UpdateLockFile, v); + evalSettings.pureEval ? AllPure : UseUpdatedLockFile, v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 132439b93..6f91686a6 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -36,16 +36,23 @@ struct LockFile }; std::map<FlakeRef, FlakeEntry> flakeEntries; - std::map<FlakeId, NonFlakeEntry> nonFlakeEntries; + std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; }; typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; Path getUserRegistryPath(); -enum ShouldUpdateLockFile { DontUpdate, UpdateLockFile, RecreateLockFile}; +enum HandleLockFile + { AllPure // Everything is handled 100% purely + , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely + , UpdateLockFile // Update the existing lockfile and write it to file + , UseUpdatedLockFile // `UpdateLockFile` without writing to file + , RecreateLockFile // Recreate the lockfile from scratch and write it to file + , UseNewLockFile // `RecreateLockFile` without writing to file + }; -void makeFlakeValue(EvalState &, const FlakeRef &, ShouldUpdateLockFile, Value &); +void makeFlakeValue(EvalState &, const FlakeRef &, HandleLockFile, Value &); std::shared_ptr<FlakeRegistry> readRegistry(const Path &); @@ -98,12 +105,12 @@ Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct ResolvedFlake { Flake flake; - std::vector<ResolvedFlake> flakeDeps; // The flake dependencies + std::map<FlakeRef, ResolvedFlake> flakeDeps; // The key in this map, is the originalRef as written in flake.nix std::vector<NonFlake> nonFlakeDeps; ResolvedFlake(const Flake & flake) : flake(flake) {} }; -ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, ShouldUpdateLockFile); +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); void updateLockFile(EvalState &, const FlakeUri &, bool recreateLockFile); diff --git a/src/nix/command.hh b/src/nix/command.hh index 32a5047a8..30d869b19 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -80,6 +80,10 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs bool recreateLockFile = false; + bool saveLockFile = true; + + bool noRegistries = false; + ref<EvalState> getEvalState(); std::vector<std::shared_ptr<Installable>> parseInstallables( diff --git a/src/nix/flake.cc b/src/nix/flake.cc index fc0fc76b4..bfb3178ad 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -126,8 +126,8 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs for (NonFlake & nonFlake : resFlake.nonFlakeDeps) printNonFlakeInfo(nonFlake, json); - for (ResolvedFlake & newResFlake : resFlake.flakeDeps) - todo.push(newResFlake); + for (auto info : resFlake.flakeDeps) + todo.push(info.second); } } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 25f3f4f9d..a2a55d949 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -26,6 +26,16 @@ SourceExprCommand::SourceExprCommand() .longName("recreate-lock-file") .description("recreate lock file from scratch") .set(&recreateLockFile, true); + + mkFlag() + .longName("dont-save-lock-file") + .description("save the newly generated lock file") + .set(&saveLockFile, false); + + mkFlag() + .longName("no-registries") + .description("don't use flake registries") + .set(&noRegistries, true); } ref<EvalState> SourceExprCommand::getEvalState() @@ -158,10 +168,12 @@ struct InstallableFlake : InstallableValue Value * toValue(EvalState & state) override { auto vFlake = state.allocValue(); - if (std::get_if<FlakeRef::IsPath>(&flakeRef.data)) - updateLockFile(state, flakeRef.to_string(), cmd.recreateLockFile); - makeFlakeValue(state, flakeRef, cmd.recreateLockFile ? RecreateLockFile : UpdateLockFile, *vFlake); + HandleLockFile handle = cmd.noRegistries ? AllPure : + cmd.recreateLockFile ? + (cmd.saveLockFile ? RecreateLockFile : UseNewLockFile) + : (cmd.saveLockFile ? UpdateLockFile : UseUpdatedLockFile); + makeFlakeValue(state, flakeRef, handle, *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; diff --git a/tests/config.nix b/tests/config.nix deleted file mode 100644 index 03810d57a..000000000 --- a/tests/config.nix +++ /dev/null @@ -1,20 +0,0 @@ -with import <nix/config.nix>; - -rec { - inherit shell; - - path = coreutils; - - system = "x86_64-linux"; - - shared = builtins.getEnv "_NIX_TEST_SHARED"; - - mkDerivation = args: - derivation ({ - inherit system; - builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - PATH = path; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; -} diff --git a/tests/flakes.sh b/tests/flakes.sh index d720eaf23..45c5f2048 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -118,7 +118,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1: [[ -e $TEST_ROOT/result/hello ]] # Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar) +(! nix eval "(builtins.getFlake "$flake2Dir")") # But should succeed in impure mode. nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure @@ -129,8 +129,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar git -C $flake2Dir commit flake.lock -m 'Add flake.lock' # Rerunning the build should not change the lockfile. -#nix build -o $TEST_ROOT/result --flake-registry $registry $flake2:bar -#[[ -z $(git -C $flake2 diff) ]] +nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar +[[ -z $(git -C $flake2Dir diff master) ]] # Now we should be able to build the flake in pure mode. nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar @@ -140,3 +140,32 @@ nix build -o $TEST_ROOT/result file://$flake2Dir:bar # Test whether indirect dependencies work. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy + +# Add dependency to flake3 +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix <<EOF +{ + name = "flake3"; + + epoch = 2019; + + requires = [ "flake1" "flake2" ]; + + description = "Fnord"; + + provides = deps: rec { + packages.xyzzy = deps.flake2.provides.packages.bar; + packages.sth = deps.flake1.provides.packages.foo; + }; +} +EOF + +git -C $flake3Dir add flake.nix +git -C $flake3Dir commit -m 'Update flake.nix' + +# Check whether `nix build` works with an incomplete lockfile +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth + +# Check whether it saved the lockfile +[[ ! (-z $(git -C $flake3Dir diff master)) ]] From 14c877b4ab097667734b050a79b45658dcf2695d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 19 Apr 2019 14:41:59 +0200 Subject: [PATCH 0224/2548] fetchGit -> fetchTarball (cherry picked from commit cbfdea685764bf66443a999e672656c54289b8c9) --- release.nix | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/release.nix b/release.nix index ab13451ff..78b39108f 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: diff --git a/shell.nix b/shell.nix index 73e75fb29..8167f87a2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; From 20a1a65d377cb49dd0a1b60a07623760bfc730d1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 21 May 2019 14:55:43 +0200 Subject: [PATCH 0225/2548] Only rewrite the lockfile if it changed This removes spurious warnings about failure to write the lockfile. --- src/libexpr/primops/flake.cc | 28 ++++++++++++++++------------ src/libexpr/primops/flake.hh | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3cbb0c1ef..788977c72 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -465,31 +465,35 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); - LockFile lockFile; + LockFile oldLockFile; if (!recreateLockFile (handleLockFile)) { // If recreateLockFile, start with an empty lockfile - lockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack + oldLockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack } + LockFile lockFile(oldLockFile); + ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); - if (allowedToWrite(handleLockFile)) { - if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { - writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + if (!(lockFile == oldLockFile)) { + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { + writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); - // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. - runProgram("git", true, { "-C", refData->path, "add", - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); - } else std::cout << "Cannot write lockfile because the FlakeRef isn't of the form IsPath." << std::endl; - } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) - std::cout << "Using updating lockfile without writing it to file" << std::endl; + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else std::cout << "Cannot write lockfile because the FlakeRef isn't of the form IsPath." << std::endl; + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + std::cout << "Using updating lockfile without writing it to file" << std::endl; + } return resFlake; } -void updateLockFile (EvalState & state, const FlakeUri & flakeUri, bool recreateLockFile) +void updateLockFile(EvalState & state, const FlakeUri & flakeUri, bool recreateLockFile) { FlakeRef flakeRef(flakeUri); resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 6f91686a6..e43b860ee 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -24,6 +24,11 @@ struct LockFile FlakeRef ref; Hash contentHash; NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; + + bool operator ==(const NonFlakeEntry & other) const + { + return ref == other.ref && contentHash == other.contentHash; + } }; struct FlakeEntry @@ -33,10 +38,26 @@ struct LockFile std::map<FlakeRef, FlakeEntry> flakeEntries; std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; + + bool operator ==(const FlakeEntry & other) const + { + return + ref == other.ref + && contentHash == other.contentHash + && flakeEntries == other.flakeEntries + && nonFlakeEntries == other.nonFlakeEntries; + } }; std::map<FlakeRef, FlakeEntry> flakeEntries; std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; + + bool operator ==(const LockFile & other) const + { + return + flakeEntries == other.flakeEntries + && nonFlakeEntries == other.nonFlakeEntries; + } }; typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; From 5990b86391a7b51879c865eed39ee46f39cbdd11 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 21 May 2019 15:03:54 +0200 Subject: [PATCH 0226/2548] Use warn(), tweak messages --- src/libexpr/primops/flake.cc | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 788977c72..6998536ec 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -387,29 +387,17 @@ LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlak return entry; } -bool allowedToWrite (HandleLockFile handle) +bool allowedToWrite(HandleLockFile handle) { - if (handle == AllPure) return false; - else if (handle == TopRefUsesRegistries) return false; - else if (handle == UpdateLockFile) return true; - else if (handle == UseUpdatedLockFile) return false; - else if (handle == RecreateLockFile) return true; - else if (handle == UseNewLockFile) return false; - else assert(false); + return handle == UpdateLockFile || handle == RecreateLockFile; } -bool recreateLockFile (HandleLockFile handle) +bool recreateLockFile(HandleLockFile handle) { - if (handle == AllPure) return false; - else if (handle == TopRefUsesRegistries) return false; - else if (handle == UpdateLockFile) return false; - else if (handle == UseUpdatedLockFile) return false; - else if (handle == RecreateLockFile) return true; - else if (handle == UseNewLockFile) return true; - else assert(false); + return handle == RecreateLockFile || handle == UseNewLockFile; } -bool allowedToUseRegistries (HandleLockFile handle, bool isTopRef) +bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) { if (handle == AllPure) return false; else if (handle == TopRefUsesRegistries) return isTopRef; @@ -433,11 +421,11 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake if (i != lockFile.nonFlakeEntries.end()) { NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); if (nonFlake.hash != i->second.contentHash) - throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); + throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.nonFlakeDeps.push_back(nonFlake); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("the lockfile requires updating nonflake dependency %s in AllPure mode", nonFlakeInfo.first); + throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); } } @@ -447,11 +435,11 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); if (newResFlake.flake.hash != i->second.contentHash) - throw Error("the content hash of flakeref %s doesn't match", i->second.ref.to_string()); + throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("the lockfile requires updating flake dependency %s in AllPure mode", newFlakeRef.to_string()); + throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); } } @@ -485,9 +473,10 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. runProgram("git", true, { "-C", refData->path, "add", (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); - } else std::cout << "Cannot write lockfile because the FlakeRef isn't of the form IsPath." << std::endl; + } else + warn("cannot write lockfile of remote flake '%s'", topRef); } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) - std::cout << "Using updating lockfile without writing it to file" << std::endl; + warn("using updated lockfile without writing it to file"); } return resFlake; From b502b6682b9c73ae5760967eaf7161b3e8523e9e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch <maximilian@mbosch.me> Date: Mon, 20 May 2019 19:17:18 +0200 Subject: [PATCH 0227/2548] doc: clarify that optional attrs in a function argument will be ignored unless specified In `args@{ a ? 1 }: /* ... */` the value `a` won't be a part of `args` unless it's specified when calling the function, the default value will be ignored in this case. My personal point of view is that this behavior is a matter of taste, at least I was pretty sure that unmatched arguments will be a part of `args@` while debugging some Nix code last week. I decided to add a warning to the docs which hopefully reduces the confusion of further Nix developers who thought the same about `args@`. --- .../expressions/language-constructs.xml | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index f961ed921..923b5d8c4 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -217,7 +217,25 @@ but can also be written as: ellipsis(<literal>...</literal>) as you can access attribute names as <literal>a</literal>, using <literal>args.a</literal>, which was given as an additional attribute to the function. - </para></listitem> + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> </itemizedlist> From 2468672e305faf672c3901c1a9605ca1cb175908 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 16 May 2019 22:48:16 +0200 Subject: [PATCH 0228/2548] Improve FlakeCommand It now handles commonality like calling getFlake() and resolving relative local flake refs. Fixes #2822. --- src/libexpr/primops/flake.cc | 11 +++--- src/libexpr/primops/flake.hh | 5 +-- src/nix/command.hh | 37 ++++++------------- src/nix/flake.cc | 69 ++++++++++++++++++++++++------------ src/nix/installables.cc | 2 +- 5 files changed, 65 insertions(+), 59 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 6998536ec..c08c30c9c 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -482,9 +482,8 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc return resFlake; } -void updateLockFile(EvalState & state, const FlakeUri & flakeUri, bool recreateLockFile) +void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) { - FlakeRef flakeRef(flakeUri); resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); } @@ -551,10 +550,8 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va static RegisterPrimOp r2("getFlake", 1, prim_getFlake); -void gitCloneFlake (std::string flakeUri, EvalState & state, Registries registries, - Path endDirectory) +void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) { - FlakeRef flakeRef(flakeUri); flakeRef = lookupFlake(state, flakeRef, registries); std::string uri; @@ -576,8 +573,8 @@ void gitCloneFlake (std::string flakeUri, EvalState & state, Registries registri } } - if (endDirectory != "") - args.push_back(endDirectory); + if (destDir != "") + args.push_back(destDir); runProgram("git", true, args); } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index e43b860ee..677cdb7b7 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -133,7 +133,8 @@ struct ResolvedFlake ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); -void updateLockFile(EvalState &, const FlakeUri &, bool recreateLockFile); +void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); + +void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); -void gitCloneFlake (std::string flakeUri, EvalState &, Registries, Path); } diff --git a/src/nix/command.hh b/src/nix/command.hh index 30d869b19..423ac5baa 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -35,26 +35,6 @@ struct Buildable typedef std::vector<Buildable> Buildables; -struct GitRepoCommand : virtual Args -{ - std::string gitPath = absPath("."); - - GitRepoCommand () - { - expectArg("git-path", &gitPath, true); - } -}; - -struct FlakeCommand : virtual Args -{ - std::string flakeUri; - - FlakeCommand() - { - expectArg("flake-uri", &flakeUri); - } -}; - struct Installable { virtual std::string what() = 0; @@ -72,7 +52,16 @@ struct Installable } }; -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +struct EvalCommand : virtual StoreCommand, MixEvalArgs +{ + ref<EvalState> getEvalState(); + +private: + + std::shared_ptr<EvalState> evalState; +}; + +struct SourceExprCommand : virtual Args, EvalCommand { std::optional<Path> file; @@ -84,8 +73,6 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs bool noRegistries = false; - ref<EvalState> getEvalState(); - std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -96,10 +83,6 @@ struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { return {"defaultPackage"}; } - -private: - - std::shared_ptr<EvalState> evalState; }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index bfb3178ad..bc2f1cb5b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -10,7 +10,33 @@ using namespace nix; -struct CmdFlakeList : StoreCommand, MixEvalArgs +class FlakeCommand : virtual Args, public EvalCommand +{ + std::string flakeUri = "."; + +public: + + FlakeCommand() + { + expectArg("flake-uri", &flakeUri, true); + } + + FlakeRef getFlakeRef() + { + if (flakeUri.find('/') != std::string::npos || flakeUri == ".") + return FlakeRef(flakeUri, true); + else + return FlakeRef(flakeUri); + } + + Flake getFlake() + { + auto evalState = getEvalState(); + return nix::getFlake(*evalState, getFlakeRef(), true); + } +}; + +struct CmdFlakeList : EvalCommand { std::string name() override { @@ -24,9 +50,7 @@ struct CmdFlakeList : StoreCommand, MixEvalArgs void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); - - auto registries = evalState->getFlakeRegistries(); + auto registries = getEvalState()->getFlakeRegistries(); stopProgressBar(); @@ -60,7 +84,7 @@ void printFlakeInfo(Flake & flake, bool json) { std::cout << "URI: " << flake.resolvedRef.to_string() << "\n"; std::cout << "Description: " << flake.description << "\n"; if (flake.resolvedRef.ref) - std::cout << "Branch: " << *flake.resolvedRef.ref; + std::cout << "Branch: " << *flake.resolvedRef.ref << "\n"; if (flake.resolvedRef.rev) std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n"; if (flake.revCount) @@ -95,7 +119,7 @@ void printNonFlakeInfo(NonFlake & nonFlake, bool json) { } } -struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs +struct CmdFlakeDeps : FlakeCommand, MixJSON { std::string name() override { @@ -109,12 +133,10 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); + auto evalState = getEvalState(); evalState->addRegistryOverrides(registryOverrides); - FlakeRef flakeRef(flakeUri); - - ResolvedFlake resFlake = resolveFlake(*evalState, flakeRef, UpdateLockFile); + ResolvedFlake resFlake = resolveFlake(*evalState, getFlakeRef(), UpdateLockFile); std::queue<ResolvedFlake> todo; todo.push(resFlake); @@ -132,7 +154,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON, StoreCommand, MixEvalArgs } }; -struct CmdFlakeUpdate : StoreCommand, FlakeCommand, MixEvalArgs +struct CmdFlakeUpdate : FlakeCommand { std::string name() override { @@ -146,14 +168,18 @@ struct CmdFlakeUpdate : StoreCommand, FlakeCommand, MixEvalArgs void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); + auto evalState = getEvalState(); - bool recreateLockFile = true; - updateLockFile(*evalState, flakeUri, recreateLockFile); + auto flakeRef = getFlakeRef(); + + if (std::get_if<FlakeRef::IsPath>(&flakeRef.data)) + updateLockFile(*evalState, flakeRef, true); + else + throw Error("cannot update lockfile of flake '%s'", flakeRef); } }; -struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand +struct CmdFlakeInfo : FlakeCommand, MixJSON { std::string name() override { @@ -169,8 +195,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON, MixEvalArgs, StoreCommand void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); - Flake flake = getFlake(*evalState, FlakeRef(flakeUri), true); + auto flake = getFlake(); printFlakeInfo(flake, json); } }; @@ -235,7 +260,7 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command } }; -struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs +struct CmdFlakePin : virtual Args, EvalCommand { FlakeUri alias; @@ -256,7 +281,7 @@ struct CmdFlakePin : virtual Args, StoreCommand, MixEvalArgs void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); + auto evalState = getEvalState(); Path userRegistryPath = getUserRegistryPath(); FlakeRegistry userRegistry = *readRegistry(userRegistryPath); @@ -307,7 +332,7 @@ struct CmdFlakeInit : virtual Args, Command } }; -struct CmdFlakeClone : StoreCommand, FlakeCommand, MixEvalArgs +struct CmdFlakeClone : FlakeCommand { Path endDirectory = ""; @@ -328,10 +353,10 @@ struct CmdFlakeClone : StoreCommand, FlakeCommand, MixEvalArgs void run(nix::ref<nix::Store> store) override { - auto evalState = std::make_shared<EvalState>(searchPath, store); + auto evalState = getEvalState(); Registries registries = evalState->getFlakeRegistries(); - gitCloneFlake(flakeUri, *evalState, registries, endDirectory); + gitCloneFlake(getFlakeRef().to_string(), *evalState, registries, endDirectory); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a2a55d949..85ef2cb56 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -38,7 +38,7 @@ SourceExprCommand::SourceExprCommand() .set(&noRegistries, true); } -ref<EvalState> SourceExprCommand::getEvalState() +ref<EvalState> EvalCommand::getEvalState() { if (!evalState) evalState = std::make_shared<EvalState>(searchPath, getStore()); From bc0fb109a946a1c3e125a5148280a0caba2d2c9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 16 May 2019 23:14:27 +0200 Subject: [PATCH 0229/2548] Add some tests --- tests/flakes.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 45c5f2048..5137bc39a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -104,6 +104,11 @@ EOF # Test 'nix flake info'. nix flake info --flake-registry $registry flake1 | grep -q 'ID: *flake1' +# Test 'nix flake info' on a local flake. +(cd $flake1Dir && nix flake info) | grep -q 'ID: *flake1' +(cd $flake1Dir && nix flake info .) | grep -q 'ID: *flake1' +nix flake info $flake1Dir | grep -q 'ID: *flake1' + # Test 'nix flake info --json'. json=$(nix flake info --flake-registry $registry flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] From 70136a9bf46bcf5a97b63f356fefd8adabf4c23b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 13:46:07 +0200 Subject: [PATCH 0230/2548] Move flake-related flags into a separate class Also, rename --dont-save-lock-file to --no-save-lock-file and change noRegistries to useRegistries. --- src/libexpr/primops/flake.hh | 2 +- src/nix/command.hh | 21 ++++++++++------ src/nix/flake.cc | 18 ++++++++----- src/nix/installables.cc | 49 +++++++++++++++++++++--------------- 4 files changed, 56 insertions(+), 34 deletions(-) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 677cdb7b7..8eaac9d96 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -64,7 +64,7 @@ typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; Path getUserRegistryPath(); -enum HandleLockFile +enum HandleLockFile : unsigned int { AllPure // Everything is handled 100% purely , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely , UpdateLockFile // Update the existing lockfile and write it to file diff --git a/src/nix/command.hh b/src/nix/command.hh index 423ac5baa..a841b879a 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -11,8 +11,8 @@ extern std::string programPath; struct Value; class Bindings; class EvalState; - class Store; +enum HandleLockFile : unsigned int; /* A command that require a Nix store. */ struct StoreCommand : virtual Command @@ -61,17 +61,24 @@ private: std::shared_ptr<EvalState> evalState; }; -struct SourceExprCommand : virtual Args, EvalCommand +struct MixFlakeOptions : virtual Args { - std::optional<Path> file; - - SourceExprCommand(); - bool recreateLockFile = false; bool saveLockFile = true; - bool noRegistries = false; + bool useRegistries = true; + + MixFlakeOptions(); + + HandleLockFile getLockFileMode(); +}; + +struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions +{ + std::optional<Path> file; + + SourceExprCommand(); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index bc2f1cb5b..b4f0f67be 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -10,7 +10,7 @@ using namespace nix; -class FlakeCommand : virtual Args, public EvalCommand +class FlakeCommand : virtual Args, public EvalCommand, public MixFlakeOptions { std::string flakeUri = "."; @@ -32,7 +32,12 @@ public: Flake getFlake() { auto evalState = getEvalState(); - return nix::getFlake(*evalState, getFlakeRef(), true); + return nix::getFlake(*evalState, getFlakeRef(), useRegistries); + } + + ResolvedFlake resolveFlake() + { + return nix::resolveFlake(*getEvalState(), getFlakeRef(), getLockFileMode()); } }; @@ -119,6 +124,7 @@ void printNonFlakeInfo(NonFlake & nonFlake, bool json) { } } +// FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand, MixJSON { std::string name() override @@ -136,7 +142,7 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON auto evalState = getEvalState(); evalState->addRegistryOverrides(registryOverrides); - ResolvedFlake resFlake = resolveFlake(*evalState, getFlakeRef(), UpdateLockFile); + auto resFlake = resolveFlake(); std::queue<ResolvedFlake> todo; todo.push(resFlake); @@ -334,7 +340,7 @@ struct CmdFlakeInit : virtual Args, Command struct CmdFlakeClone : FlakeCommand { - Path endDirectory = ""; + Path destDir; std::string name() override { @@ -348,7 +354,7 @@ struct CmdFlakeClone : FlakeCommand CmdFlakeClone() { - expectArg("end-dir", &endDirectory, true); + expectArg("dest-dir", &destDir, true); } void run(nix::ref<nix::Store> store) override @@ -356,7 +362,7 @@ struct CmdFlakeClone : FlakeCommand auto evalState = getEvalState(); Registries registries = evalState->getFlakeRegistries(); - gitCloneFlake(getFlakeRef().to_string(), *evalState, registries, endDirectory); + gitCloneFlake(getFlakeRef().to_string(), *evalState, registries, destDir); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 85ef2cb56..1a79f49fb 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -13,6 +13,34 @@ namespace nix { +MixFlakeOptions::MixFlakeOptions() +{ + mkFlag() + .longName("recreate-lock-file") + .description("recreate lock file from scratch") + .set(&recreateLockFile, true); + + mkFlag() + .longName("no-save-lock-file") + .description("do not save the newly generated lock file") + .set(&saveLockFile, false); + + mkFlag() + .longName("no-registries") + .description("don't use flake registries") + .set(&useRegistries, false); +} + +HandleLockFile MixFlakeOptions::getLockFileMode() +{ + return + useRegistries + ? recreateLockFile + ? (saveLockFile ? RecreateLockFile : UseNewLockFile) + : (saveLockFile ? UpdateLockFile : UseUpdatedLockFile) + : AllPure; +} + SourceExprCommand::SourceExprCommand() { mkFlag() @@ -21,21 +49,6 @@ SourceExprCommand::SourceExprCommand() .label("file") .description("evaluate a set of attributes from FILE (deprecated)") .dest(&file); - - mkFlag() - .longName("recreate-lock-file") - .description("recreate lock file from scratch") - .set(&recreateLockFile, true); - - mkFlag() - .longName("dont-save-lock-file") - .description("save the newly generated lock file") - .set(&saveLockFile, false); - - mkFlag() - .longName("no-registries") - .description("don't use flake registries") - .set(&noRegistries, true); } ref<EvalState> EvalCommand::getEvalState() @@ -169,11 +182,7 @@ struct InstallableFlake : InstallableValue { auto vFlake = state.allocValue(); - HandleLockFile handle = cmd.noRegistries ? AllPure : - cmd.recreateLockFile ? - (cmd.saveLockFile ? RecreateLockFile : UseNewLockFile) - : (cmd.saveLockFile ? UpdateLockFile : UseUpdatedLockFile); - makeFlakeValue(state, flakeRef, handle, *vFlake); + makeFlakeValue(state, flakeRef, cmd.getLockFileMode(), *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; From 3e8ef9eb22a0e36ef5ecaabf414b6edd46b87858 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 13:57:19 +0200 Subject: [PATCH 0231/2548] nix flake deps: Print flake dependencies --- src/nix/flake.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b4f0f67be..ecbb3b81f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -70,7 +70,7 @@ struct CmdFlakeList : EvalCommand } }; -void printFlakeInfo(Flake & flake, bool json) { +void printFlakeInfo(const Flake & flake, bool json) { if (json) { nlohmann::json j; j["id"] = flake.id; @@ -98,7 +98,7 @@ void printFlakeInfo(Flake & flake, bool json) { } } -void printNonFlakeInfo(NonFlake & nonFlake, bool json) { +void printNonFlakeInfo(const NonFlake & nonFlake, bool json) { if (json) { nlohmann::json j; j["id"] = nonFlake.alias; @@ -142,20 +142,20 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON auto evalState = getEvalState(); evalState->addRegistryOverrides(registryOverrides); - auto resFlake = resolveFlake(); - std::queue<ResolvedFlake> todo; - todo.push(resFlake); + todo.push(resolveFlake()); while (!todo.empty()) { - resFlake = todo.front(); + auto resFlake = std::move(todo.front()); todo.pop(); - for (NonFlake & nonFlake : resFlake.nonFlakeDeps) + for (auto & nonFlake : resFlake.nonFlakeDeps) printNonFlakeInfo(nonFlake, json); - for (auto info : resFlake.flakeDeps) + for (auto & info : resFlake.flakeDeps) { + printFlakeInfo(info.second.flake, json); todo.push(info.second); + } } } }; From e414bde6f9f58e599d48307ff3cb0ab64cb47d9a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 14:31:40 +0200 Subject: [PATCH 0232/2548] Check the flake epoch Closes #2883. --- src/libexpr/primops/flake.cc | 11 +++++++++-- src/libexpr/primops/flake.hh | 3 ++- src/nix/flake.cc | 2 ++ tests/flakes.sh | 6 +++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c08c30c9c..9131080bf 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -309,10 +309,17 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.forceAttrs(vInfo); + if (auto epoch = vInfo.attrs->get(state.symbols.create("epoch"))) { + flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch > 2019) + throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + } else + throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + if (auto name = vInfo.attrs->get(state.sName)) flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); else - throw Error("flake lacks attribute 'name'"); + throw Error("flake '%s' lacks attribute 'name'", flakeRef); if (auto description = vInfo.attrs->get(state.sDescription)) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); @@ -337,7 +344,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.forceFunction(*(**provides).value, *(**provides).pos); flake.vProvides = (**provides).value; } else - throw Error("flake lacks attribute 'provides'"); + throw Error("flake '%s' lacks attribute 'provides'", flakeRef); return flake; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 8eaac9d96..983c0eab6 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -101,7 +101,8 @@ struct Flake std::map<FlakeAlias, FlakeRef> nonFlakeRequires; Value * vProvides; // FIXME: gc // date - // content hash + unsigned int epoch; + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ecbb3b81f..2dcdfc663 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -83,6 +83,7 @@ void printFlakeInfo(const Flake & flake, bool json) { if (flake.revCount) j["revCount"] = *flake.revCount; j["path"] = flake.storePath; + j["epoch"] = flake.epoch; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; @@ -95,6 +96,7 @@ void printFlakeInfo(const Flake & flake, bool json) { if (flake.revCount) std::cout << "Revcount: " << *flake.revCount << "\n"; std::cout << "Path: " << flake.storePath << "\n"; + std::cout << "Epoch: " << flake.epoch << "\n"; } } diff --git a/tests/flakes.sh b/tests/flakes.sh index 5137bc39a..6c987ad14 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -146,7 +146,7 @@ nix build -o $TEST_ROOT/result file://$flake2Dir:bar # Test whether indirect dependencies work. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy -# Add dependency to flake3 +# Add dependency to flake3. rm $flake3Dir/flake.nix cat > $flake3Dir/flake.nix <<EOF @@ -174,3 +174,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth # Check whether it saved the lockfile [[ ! (-z $(git -C $flake3Dir diff master)) ]] + +# Unsupported epochs should be an error. +sed -i $flake3Dir/flake.nix -e s/2019/2030/ +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch' From 7c7105e0f89fb7b0971fbe012296f811f245e798 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 14:52:57 +0200 Subject: [PATCH 0233/2548] flake-registry: tweag/nix -> NixOS/nix --- flake-registry.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake-registry.json b/flake-registry.json index 348bd010a..725bcef07 100644 --- a/flake-registry.json +++ b/flake-registry.json @@ -4,7 +4,7 @@ "uri": "github:edolstra/dwarffs/flake" }, "nix": { - "uri": "github:tweag/nix/flakes" + "uri": "github:NixOS/nix/flakes" }, "nixpkgs": { "uri": "github:edolstra/nixpkgs/release-19.03" From 66f1d7ee95ba693a15ae5dc413289fee954f0f04 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 22:56:46 +0200 Subject: [PATCH 0234/2548] Fetch the flake registry from the NixOS/flake-registry repo --- flake-registry.json | 20 -------------------- local.mk | 2 -- src/libexpr/eval.cc | 8 -------- src/libexpr/eval.hh | 10 +++++----- src/libexpr/primops/flake.cc | 16 +++++++++++++--- src/libexpr/primops/flake.hh | 2 -- src/nix/flake.cc | 2 +- 7 files changed, 19 insertions(+), 41 deletions(-) delete mode 100644 flake-registry.json diff --git a/flake-registry.json b/flake-registry.json deleted file mode 100644 index 725bcef07..000000000 --- a/flake-registry.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "flakes": { - "dwarffs": { - "uri": "github:edolstra/dwarffs/flake" - }, - "nix": { - "uri": "github:NixOS/nix/flakes" - }, - "nixpkgs": { - "uri": "github:edolstra/nixpkgs/release-19.03" - }, - "hydra": { - "uri": "github:NixOS/hydra/flake" - }, - "patchelf": { - "uri": "github:NixOS/patchelf" - } - }, - "version": 1 -} diff --git a/local.mk b/local.mk index 11ed9c0a6..4b380176f 100644 --- a/local.mk +++ b/local.mk @@ -10,5 +10,3 @@ GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) - -$(eval $(call install-data-in,$(d)/flake-registry.json,$(datadir)/nix)) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 2789ea313..0f8a105b1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1978,14 +1978,6 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings evalSettings; -EvalSettings::EvalSettings() -{ - if (flakeRegistry == "") - // FIXME: static initialization order fiasco. But this will go - // away when we switch to an online registry. - flakeRegistry = settings.nixDataDir + "/nix/flake-registry.json"; -} - static GlobalConfig::Register r1(&evalSettings); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b0bf777fc..1e45bc1a8 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -325,9 +325,11 @@ public: const std::vector<std::shared_ptr<FlakeRegistry>> getFlakeRegistries(); + std::shared_ptr<FlakeRegistry> getGlobalFlakeRegistry(); + private: - std::shared_ptr<FlakeRegistry> _flakeRegistry; - std::once_flag _flakeRegistryInit; + std::shared_ptr<FlakeRegistry> _globalFlakeRegistry; + std::once_flag _globalFlakeRegistryInit; }; @@ -368,10 +370,8 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; - Setting<std::string> flakeRegistry{this, "", "flake-registry", + Setting<std::string> flakeRegistry{this, "https://raw.githubusercontent.com/NixOS/flake-registry/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; - - EvalSettings(); }; extern EvalSettings evalSettings; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 9131080bf..c6c380118 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -131,9 +131,19 @@ void writeLockFile(const LockFile & lockFile, const Path & path) writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file } -std::shared_ptr<FlakeRegistry> getGlobalRegistry() +std::shared_ptr<FlakeRegistry> EvalState::getGlobalFlakeRegistry() { - return readRegistry(evalSettings.flakeRegistry); + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) + path = getDownloader()->downloadCached(store, + evalSettings.flakeRegistry, false, "registry").path; + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; } Path getUserRegistryPath() @@ -162,7 +172,7 @@ const Registries EvalState::getFlakeRegistries() Registries registries; registries.push_back(getFlagRegistry(registryOverrides)); registries.push_back(getUserRegistry()); - registries.push_back(getGlobalRegistry()); + registries.push_back(getGlobalFlakeRegistry()); return registries; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 983c0eab6..b965aa9e7 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -120,8 +120,6 @@ struct NonFlake resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; }; -std::shared_ptr<FlakeRegistry> getGlobalRegistry(); - Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct ResolvedFlake diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2dcdfc663..912b154c1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -298,7 +298,7 @@ struct CmdFlakePin : virtual Args, EvalCommand it->second = getFlake(*evalState, it->second, true).resolvedRef; writeRegistry(userRegistry, userRegistryPath); } else { - std::shared_ptr<FlakeRegistry> globalReg = getGlobalRegistry(); + std::shared_ptr<FlakeRegistry> globalReg = evalState->getGlobalFlakeRegistry(); it = globalReg->entries.find(FlakeRef(alias)); if (it != globalReg->entries.end()) { FlakeRef newRef = getFlake(*evalState, it->second, true).resolvedRef; From df3f5a78d5ab0a1f2dc9d288b271b38a9b8b33b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 23:36:29 +0200 Subject: [PATCH 0235/2548] Refactor downloadCached() interface --- src/libexpr/common-eval-args.cc | 8 +++--- src/libexpr/parser.y | 4 ++- src/libexpr/primops.cc | 22 ++++++++--------- src/libexpr/primops/flake.cc | 15 +++++++---- src/libstore/download.cc | 44 ++++++++++++++++----------------- src/libstore/download.hh | 20 +++++++++++---- src/nix-channel/nix-channel.cc | 14 +++++------ 7 files changed, 73 insertions(+), 54 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 8e94d358e..7c0d268bd 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -61,9 +61,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true).path; - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 75f04df3e..0cfe29c96 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -657,7 +657,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 55a1bde11..070e72f3a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index c6c380118..e2fdf08ca 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -136,9 +136,11 @@ std::shared_ptr<FlakeRegistry> EvalState::getGlobalFlakeRegistry() std::call_once(_globalFlakeRegistryInit, [&]() { auto path = evalSettings.flakeRegistry; - if (!hasPrefix(path, "/")) - path = getDownloader()->downloadCached(store, - evalSettings.flakeRegistry, false, "registry").path; + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + path = getDownloader()->downloadCached(store, request).path; + } _globalFlakeRegistry = readRegistry(path); }); @@ -244,8 +246,11 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool if (accessToken != "") url += "?access_token=" + accessToken; - auto result = getDownloader()->downloadCached(state.store, url, true, "source", - Hash(), nullptr, resolvedRef.rev ? 1000000000 : settings.tarballTtl); + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + auto result = getDownloader()->downloadCached(state.store, request); if (!result.etag) throw Error("did not receive an ETag header from '%s'", url); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 975cfd97d..a7c2600f6 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -319,10 +319,10 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); @@ -790,18 +790,20 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref<Store> store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; result.storePath = expectedStorePath; @@ -835,10 +837,9 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); @@ -852,18 +853,17 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -883,7 +883,7 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -906,11 +906,11 @@ CachedDownloadResult Downloader::downloadCached(ref<Store> store, const string & } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + url, request.expectedHash.to_string(), gotHash.to_string()); } result.storePath = storePath; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index aa8c34be2..b676a1a7b 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -36,11 +36,23 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr<std::string> data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + struct CachedDownloadResult { // Note: 'storePath' may be different from 'path' when using a @@ -48,6 +60,7 @@ struct CachedDownloadResult Path storePath; Path path; std::optional<std::string> etag; + std::string effectiveUri; }; class Store; @@ -73,10 +86,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached( - ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, - int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 7b23088a2..bd1371dba 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -121,12 +123,10 @@ static void update(const StringSet & channelNames) } // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } From f0d6d67af93b63c1da1809dc7630026624c19b14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 23:43:58 +0200 Subject: [PATCH 0236/2548] Prevent the global registry from being GC'ed Issue #2868. --- src/libexpr/primops/flake.cc | 1 + src/libstore/download.cc | 3 +++ src/libstore/download.hh | 1 + 3 files changed, 5 insertions(+) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e2fdf08ca..3c3d5e0c7 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -139,6 +139,7 @@ std::shared_ptr<FlakeRegistry> EvalState::getGlobalFlakeRegistry() if (!hasPrefix(path, "/")) { CachedDownloadRequest request(evalSettings.flakeRegistry); request.name = "flake-registry.json"; + request.gcRoot = true; path = getDownloader()->downloadCached(store, request).path; } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index a7c2600f6..0d1974d3b 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -913,6 +913,9 @@ CachedDownloadResult Downloader::downloadCached( url, request.expectedHash.to_string(), gotHash.to_string()); } + if (request.gcRoot) + store->addIndirectRoot(fileLink); + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index b676a1a7b..404e51195 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -48,6 +48,7 @@ struct CachedDownloadRequest std::string name; Hash expectedHash; unsigned int ttl = settings.tarballTtl; + bool gcRoot = false; CachedDownloadRequest(const std::string & uri) : uri(uri) { } From a4ba6e5590285a78e51079be7664a459b2ae7ee4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 22 May 2019 23:52:29 +0200 Subject: [PATCH 0237/2548] Add a test for the registry GC root --- tests/flakes.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 6c987ad14..977728e43 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -178,3 +178,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth # Unsupported epochs should be an error. sed -i $flake3Dir/flake.nix -e s/2019/2030/ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch' + +# Test whether registry caching works. +nix flake list --flake-registry file://$registry | grep -q flake3 +mv $registry $registry.tmp +nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake3 From 013f4928c8a6dd21685531ceb631fbcff85cefef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 23 May 2019 16:36:12 +0200 Subject: [PATCH 0238/2548] Fix tests https://hydra.nixos.org/eval/1521131 --- tests/flakes.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 977728e43..822c94e42 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -105,9 +105,9 @@ EOF nix flake info --flake-registry $registry flake1 | grep -q 'ID: *flake1' # Test 'nix flake info' on a local flake. -(cd $flake1Dir && nix flake info) | grep -q 'ID: *flake1' -(cd $flake1Dir && nix flake info .) | grep -q 'ID: *flake1' -nix flake info $flake1Dir | grep -q 'ID: *flake1' +(cd $flake1Dir && nix flake info --flake-registry $registry) | grep -q 'ID: *flake1' +(cd $flake1Dir && nix flake info --flake-registry $registry .) | grep -q 'ID: *flake1' +nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1' # Test 'nix flake info --json'. json=$(nix flake info --flake-registry $registry flake1 --json | jq .) @@ -141,7 +141,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar # Or without a registry. -nix build -o $TEST_ROOT/result file://$flake2Dir:bar +# FIXME: shouldn't need '--flake-registry /no-registry'? +nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar # Test whether indirect dependencies work. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy From 6e984431dd27326681a7cbb56404665c353c834a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 23 May 2019 23:38:40 +0200 Subject: [PATCH 0239/2548] fetchGit: Don't barf if we can't update our Git clone Instead print a warning that we're continuing with the most recently fetched version. --- src/libexpr/primops/fetchGit.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 00bbeb6d8..f6b096c4a 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -139,7 +139,12 @@ GitInfo exportGit(ref<Store> store, std::string uri, // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", uri); + } struct timeval times[2]; times[0].tv_sec = now; From 6b77bfc28d36d5b8111e23e15fbe4513a0797f47 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 23 May 2019 23:39:58 +0200 Subject: [PATCH 0240/2548] FlakeRef::to_string(): Check round trip --- src/libexpr/primops/flakeref.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 3842b3f1a..c42616374 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -194,6 +194,8 @@ std::string FlakeRef::to_string() const else abort(); + assert(FlakeRef(string) == *this); + return string; } From 90fe1dfd2fd85f17609166c92a2163dfaa09ea99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Thu, 23 May 2019 23:42:13 +0200 Subject: [PATCH 0241/2548] Register flake source trees as GC roots This ensures that flakes don't get garbage-collected, which is important to get nix-channel-like behaviour. For example, running $ nix build hydra: will create a GC root ~/.cache/nix/flake-closures/hydra -> /nix/store/xarfiqcwa4w8r4qpz1a769xxs8c3phgn-flake-closure where the contents/references of the linked file in the store are the flake source trees used by the 'hydra' flake: /nix/store/n6d5f5lkpfjbmkyby0nlg8y1wbkmbc7i-source /nix/store/vbkg4zy1qd29fnhflsv9k2j9jnbqd5m2-source /nix/store/z46xni7d47s5wk694359mq9ay353ar94-source Note that this in itself is not enough to allow offline use; the fetcher for the flakeref (e.g. fetchGit or downloadCached) must not fail if it cannot fetch the latest version of the file, so long as it knows a cached version. Issue #2868. --- src/libexpr/primops/flake.hh | 2 ++ src/nix/installables.cc | 45 +++++++++++++++++++++++++++++++++++- tests/flakes.sh | 19 +++++++++++++-- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index b965aa9e7..f85e62e7f 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -132,6 +132,8 @@ struct ResolvedFlake ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); +void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v); + void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1a79f49fb..6cab06240 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -10,6 +10,7 @@ #include "primops/flake.hh" #include <regex> +#include <queue> namespace nix { @@ -162,6 +163,44 @@ struct InstallableAttrPath : InstallableValue } }; +void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const ResolvedFlake & resFlake) +{ + if (std::get_if<FlakeRef::IsPath>(&origFlakeRef.data)) return; + + /* Get the store paths of all non-local flakes. */ + PathSet closure; + + std::queue<std::reference_wrapper<const ResolvedFlake>> queue; + queue.push(resFlake); + + while (!queue.empty()) { + const ResolvedFlake & flake = queue.front(); + queue.pop(); + if (!std::get_if<FlakeRef::IsPath>(&flake.flake.resolvedRef.data)) + closure.insert(flake.flake.storePath); + for (const auto & dep : flake.flakeDeps) + queue.push(dep.second); + } + + if (closure.empty()) return; + + /* Write the closure to a file in the store. */ + auto closurePath = store.addTextToStore("flake-closure", concatStringsSep(" ", closure), closure); + + Path cacheDir = getCacheDir() + "/nix/flake-closures"; + createDirs(cacheDir); + + auto s = origFlakeRef.to_string(); + assert(s[0] != '.'); + s = replaceStrings(s, "%", "%25"); + s = replaceStrings(s, "/", "%2f"); + s = replaceStrings(s, ":", "%3a"); + Path symlink = cacheDir + "/" + s; + debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); + replaceSymlink(closurePath, symlink); + store.addIndirectRoot(symlink); +} + struct InstallableFlake : InstallableValue { FlakeRef flakeRef; @@ -182,7 +221,11 @@ struct InstallableFlake : InstallableValue { auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, cmd.getLockFileMode(), *vFlake); + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + callFlake(state, resFlake, *vFlake); + + makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; diff --git a/tests/flakes.sh b/tests/flakes.sh index 822c94e42..179fef320 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -5,7 +5,10 @@ if [[ -z $(type -p git) ]]; then exit 99 fi +export _NIX_FORCE_HTTP=1 + clearStore +rm -rf $TEST_HOME/.cache registry=$TEST_ROOT/registry.json @@ -14,7 +17,7 @@ flake2Dir=$TEST_ROOT/flake2 flake3Dir=$TEST_ROOT/flake3 for repo in $flake1Dir $flake2Dir $flake3Dir; do - rm -rf $repo + rm -rf $repo $repo.tmp mkdir $repo git -C $repo init git -C $repo config user.email "foobar@example.com" @@ -142,7 +145,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar # Or without a registry. # FIXME: shouldn't need '--flake-registry /no-registry'? -nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar --tarball-ttl 0 # Test whether indirect dependencies work. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy @@ -184,3 +187,15 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | nix flake list --flake-registry file://$registry | grep -q flake3 mv $registry $registry.tmp nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake3 +mv $registry.tmp $registry + +# Test whether flakes are registered as GC roots for offline use. +rm -rf $TEST_HOME/.cache +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar +mv $flake1Dir $flake1Dir.tmp +mv $flake2Dir $flake2Dir.tmp +nix-store --gc +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar --tarball-ttl 0 +mv $flake1Dir.tmp $flake1Dir +mv $flake2Dir.tmp $flake2Dir From 638c56caeddbb10eeb7c636f7c0f4562ef6756e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Sun, 26 May 2019 15:59:50 +0200 Subject: [PATCH 0242/2548] Remove outdated fetchGit test It's no longer an error if we can't update our clone. --- tests/fetchGit.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index d87ce8560..be46d24a7 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -50,9 +50,6 @@ path2=$(nix eval --impure --raw "(builtins.fetchGit file://$repo).outPath") [[ $(nix eval --impure "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --impure --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -# But with TTL 0, it should fail. -(! nix eval --impure --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) - # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] From 6fb7545fa19fe139305972020ee6ab189c97cd6e Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Tue, 28 May 2019 10:17:28 +0200 Subject: [PATCH 0243/2548] Fixed relative path parsing Fixed #2821 --- src/libexpr/primops/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index c42616374..3c805eff8 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -134,7 +134,7 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) data = d; } - else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || uri == "."))) { + else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) { IsPath d; d.path = allowRelative ? absPath(uri) : canonPath(uri); data = d; From 4d030a8d96a3baf7855d840f0ca475d43cbdb18a Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Tue, 28 May 2019 10:51:45 +0200 Subject: [PATCH 0244/2548] Added nonFlakeRequires test Fixes #2888 --- src/libexpr/primops/flake.cc | 6 +++--- tests/flakes.sh | 40 +++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 3c3d5e0c7..d82c2389d 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -366,9 +366,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } // Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) { - SourceInfo sourceInfo = fetchFlake(state, flakeRef); + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; @@ -449,7 +449,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); - deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first)); + deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false))); } } diff --git a/tests/flakes.sh b/tests/flakes.sh index 179fef320..6081e8939 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -15,8 +15,9 @@ registry=$TEST_ROOT/registry.json flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 flake3Dir=$TEST_ROOT/flake3 +nonFlakeDir=$TEST_ROOT/nonFlake -for repo in $flake1Dir $flake2Dir $flake3Dir; do +for repo in $flake1Dir $flake2Dir $flake3Dir $nonFlakeDir; do rm -rf $repo $repo.tmp mkdir $repo git -C $repo init @@ -81,6 +82,13 @@ EOF git -C $flake3Dir add flake.nix git -C $flake3Dir commit -m 'Initial' +cat > $nonFlakeDir/README.md <<EOF +Not much +EOF + +git -C $nonFlakeDir add README.md +git -C $nonFlakeDir commit -m 'Initial' + cat > $registry <<EOF { "flakes": { @@ -199,3 +207,33 @@ nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2D nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar --tarball-ttl 0 mv $flake1Dir.tmp $flake1Dir mv $flake2Dir.tmp $flake2Dir + +# Add nonFlakeRequires to flake3. +rm $flake3Dir/flake.nix + +cat > $flake3Dir/flake.nix <<EOF +{ + name = "flake3"; + + epoch = 2019; + + requires = [ "flake1" "flake2" ]; + + nonFlakeRequires = { + nonFlake = "$nonFlakeDir"; + }; + + description = "Fnord"; + + provides = deps: rec { + packages.xyzzy = deps.flake2.provides.packages.bar; + packages.sth = deps.flake1.provides.packages.foo; + }; +} +EOF + +git -C $flake3Dir add flake.nix +git -C $flake3Dir commit -m 'Add nonFlakeRequires' + +# Check whether `nix build` works with a lockfile which is missing a nonFlakeRequires +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth From 6d7efcfaeb715b04986ef5a6b1d8f57de4ba5509 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 12:58:28 +0200 Subject: [PATCH 0245/2548] Store SourceInfo in Flake and NonFlake This deduplicates some shared fields. Factoring out the commonality is useful in places like makeFlakeValue(). --- src/libexpr/primops/flake.cc | 21 +++++++++++---------- src/libexpr/primops/flake.hh | 14 +++++--------- src/nix/flake.cc | 24 ++++++++++++------------ src/nix/installables.cc | 2 +- 4 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index d82c2389d..a8e3e0859 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -368,19 +368,19 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe // Get the `NonFlake` corresponding to a `FlakeRef`. NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) { - SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; NonFlake nonFlake(flakeRef, sourceInfo); - state.store->assertStorePath(nonFlake.storePath); + state.store->assertStorePath(nonFlake.sourceInfo.storePath); if (state.allowedPaths) - state.allowedPaths->insert(nonFlake.storePath); + state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - nonFlake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash; + nonFlake.hash = state.store->queryPathInfo(nonFlake.sourceInfo.storePath)->narHash; nonFlake.alias = alias; @@ -480,7 +480,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!recreateLockFile (handleLockFile)) { // If recreateLockFile, start with an empty lockfile - oldLockFile = readLockFile(flake.storePath + "/flake.lock"); // FIXME: symlink attack + oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack } LockFile lockFile(oldLockFile); @@ -527,15 +527,16 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) auto vNonFlake = state.allocAttr(v, nonFlake.alias); state.mkAttrs(*vNonFlake, 4); - state.store->isValidPath(nonFlake.storePath); - mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.storePath, {nonFlake.storePath}); + state.store->isValidPath(nonFlake.sourceInfo.storePath); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); // FIXME: add rev, shortRev, revCount, ... } mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); - auto & path = resFlake.flake.storePath; + auto & path = resFlake.flake.sourceInfo.storePath; state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); @@ -546,8 +547,8 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) resFlake.flake.resolvedRef.rev->gitShortRev()); } - if (resFlake.flake.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.revCount); + if (resFlake.flake.sourceInfo.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *resFlake.flake.vProvides, v); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index f85e62e7f..46489c085 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -94,17 +94,15 @@ struct Flake FlakeRef originalRef; FlakeRef resolvedRef; std::string description; - std::optional<uint64_t> revCount; - Path storePath; + SourceInfo sourceInfo; Hash hash; // content hash std::vector<FlakeRef> requires; std::map<FlakeAlias, FlakeRef> nonFlakeRequires; Value * vProvides; // FIXME: gc - // date unsigned int epoch; Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), - resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; + resolvedRef(sourceInfo.resolvedRef), sourceInfo(sourceInfo) {}; }; struct NonFlake @@ -112,12 +110,10 @@ struct NonFlake FlakeAlias alias; FlakeRef originalRef; FlakeRef resolvedRef; - std::optional<uint64_t> revCount; + SourceInfo sourceInfo; Hash hash; - Path storePath; - // date - NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), - resolvedRef(sourceInfo.resolvedRef), revCount(sourceInfo.revCount), storePath(sourceInfo.storePath) {}; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : + originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), sourceInfo(sourceInfo) {}; }; Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 912b154c1..71a6c16d9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -80,9 +80,9 @@ void printFlakeInfo(const Flake & flake, bool json) { j["branch"] = *flake.resolvedRef.ref; if (flake.resolvedRef.rev) j["revision"] = flake.resolvedRef.rev->to_string(Base16, false); - if (flake.revCount) - j["revCount"] = *flake.revCount; - j["path"] = flake.storePath; + if (flake.sourceInfo.revCount) + j["revCount"] = *flake.sourceInfo.revCount; + j["path"] = flake.sourceInfo.storePath; j["epoch"] = flake.epoch; std::cout << j.dump(4) << std::endl; } else { @@ -93,9 +93,9 @@ void printFlakeInfo(const Flake & flake, bool json) { std::cout << "Branch: " << *flake.resolvedRef.ref << "\n"; if (flake.resolvedRef.rev) std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n"; - if (flake.revCount) - std::cout << "Revcount: " << *flake.revCount << "\n"; - std::cout << "Path: " << flake.storePath << "\n"; + if (flake.sourceInfo.revCount) + std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; + std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; std::cout << "Epoch: " << flake.epoch << "\n"; } } @@ -109,9 +109,9 @@ void printNonFlakeInfo(const NonFlake & nonFlake, bool json) { j["branch"] = *nonFlake.resolvedRef.ref; if (nonFlake.resolvedRef.rev) j["revision"] = nonFlake.resolvedRef.rev->to_string(Base16, false); - if (nonFlake.revCount) - j["revCount"] = *nonFlake.revCount; - j["path"] = nonFlake.storePath; + if (nonFlake.sourceInfo.revCount) + j["revCount"] = *nonFlake.sourceInfo.revCount; + j["path"] = nonFlake.sourceInfo.storePath; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << nonFlake.alias << "\n"; @@ -120,9 +120,9 @@ void printNonFlakeInfo(const NonFlake & nonFlake, bool json) { std::cout << "Branch: " << *nonFlake.resolvedRef.ref; if (nonFlake.resolvedRef.rev) std::cout << "Revision: " << nonFlake.resolvedRef.rev->to_string(Base16, false) << "\n"; - if (nonFlake.revCount) - std::cout << "Revcount: " << *nonFlake.revCount << "\n"; - std::cout << "Path: " << nonFlake.storePath << "\n"; + if (nonFlake.sourceInfo.revCount) + std::cout << "Revcount: " << *nonFlake.sourceInfo.revCount << "\n"; + std::cout << "Path: " << nonFlake.sourceInfo.storePath << "\n"; } } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 6cab06240..ce09a43d0 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -177,7 +177,7 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const const ResolvedFlake & flake = queue.front(); queue.pop(); if (!std::get_if<FlakeRef::IsPath>(&flake.flake.resolvedRef.data)) - closure.insert(flake.flake.storePath); + closure.insert(flake.flake.sourceInfo.storePath); for (const auto & dep : flake.flakeDeps) queue.push(dep.second); } From 894e0074453d58eb3e1902c3de3d483538e2a026 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:07:15 +0200 Subject: [PATCH 0246/2548] Move hash into SourceInfo and rename to narHash to avoid ambiguity --- src/libexpr/primops/flake.cc | 14 +++++++------- src/libexpr/primops/flake.hh | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index a8e3e0859..ff3cd8184 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -263,6 +263,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); SourceInfo info(ref); info.storePath = result.storePath; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } @@ -276,6 +277,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } @@ -289,6 +291,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; return info; } @@ -315,7 +318,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); Flake flake(flakeRef, sourceInfo); - flake.hash = state.store->queryPathInfo(sourceInfo.storePath)->narHash; if (!pathExists(realFlakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); @@ -380,8 +382,6 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al if (state.allowedPaths) state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - nonFlake.hash = state.store->queryPathInfo(nonFlake.sourceInfo.storePath)->narHash; - nonFlake.alias = alias; return nonFlake; @@ -397,13 +397,13 @@ LockFile entryToLockFile(const LockFile::FlakeEntry & entry) LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.hash); + LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.sourceInfo.narHash); for (auto & info : resolvedFlake.flakeDeps) entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { - LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.hash); + LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.sourceInfo.narHash); entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); } @@ -443,7 +443,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); if (i != lockFile.nonFlakeEntries.end()) { NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); - if (nonFlake.hash != i->second.contentHash) + if (nonFlake.sourceInfo.narHash != i->second.contentHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.nonFlakeDeps.push_back(nonFlake); } else { @@ -457,7 +457,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); - if (newResFlake.flake.hash != i->second.contentHash) + if (newResFlake.flake.sourceInfo.narHash != i->second.contentHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); } else { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 46489c085..760d66057 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -84,7 +84,7 @@ struct SourceInfo FlakeRef resolvedRef; Path storePath; std::optional<uint64_t> revCount; - // date + Hash narHash; // store path hash SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; }; @@ -95,7 +95,6 @@ struct Flake FlakeRef resolvedRef; std::string description; SourceInfo sourceInfo; - Hash hash; // content hash std::vector<FlakeRef> requires; std::map<FlakeAlias, FlakeRef> nonFlakeRequires; Value * vProvides; // FIXME: gc @@ -111,7 +110,6 @@ struct NonFlake FlakeRef originalRef; FlakeRef resolvedRef; SourceInfo sourceInfo; - Hash hash; NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), sourceInfo(sourceInfo) {}; }; From 48463045415d6099b01e360d27f3cdd1053c9ed0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:08:40 +0200 Subject: [PATCH 0247/2548] Rename contentHash -> narHash for consistency --- src/libexpr/primops/flake.cc | 10 +++++----- src/libexpr/primops/flake.hh | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index ff3cd8184..d04f665a8 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -105,10 +105,10 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) { nlohmann::json json; json["uri"] = entry.ref.to_string(); - json["contentHash"] = entry.contentHash.to_string(SRI); + json["contentHash"] = entry.narHash.to_string(SRI); for (auto & x : entry.nonFlakeEntries) { json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI); + json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); } for (auto & x : entry.flakeEntries) json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); @@ -122,7 +122,7 @@ void writeLockFile(const LockFile & lockFile, const Path & path) json["nonFlakeRequires"] = nlohmann::json::object(); for (auto & x : lockFile.nonFlakeEntries) { json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeRequires"][x.first]["contentHash"] = x.second.contentHash.to_string(SRI); + json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); } json["requires"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) @@ -443,7 +443,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); if (i != lockFile.nonFlakeEntries.end()) { NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); - if (nonFlake.sourceInfo.narHash != i->second.contentHash) + if (nonFlake.sourceInfo.narHash != i->second.narHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.nonFlakeDeps.push_back(nonFlake); } else { @@ -457,7 +457,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); - if (newResFlake.flake.sourceInfo.narHash != i->second.contentHash) + if (newResFlake.flake.sourceInfo.narHash != i->second.narHash) throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); } else { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 760d66057..d90404ae6 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -22,28 +22,28 @@ struct LockFile struct NonFlakeEntry { FlakeRef ref; - Hash contentHash; - NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; + Hash narHash; + NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; bool operator ==(const NonFlakeEntry & other) const { - return ref == other.ref && contentHash == other.contentHash; + return ref == other.ref && narHash == other.narHash; } }; struct FlakeEntry { FlakeRef ref; - Hash contentHash; + Hash narHash; std::map<FlakeRef, FlakeEntry> flakeEntries; std::map<FlakeAlias, NonFlakeEntry> nonFlakeEntries; - FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), contentHash(hash) {}; + FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; bool operator ==(const FlakeEntry & other) const { return ref == other.ref - && contentHash == other.contentHash + && narHash == other.narHash && flakeEntries == other.flakeEntries && nonFlakeEntries == other.nonFlakeEntries; } From dda4f7167b9a421f1bb85ee5eed79a1bf03a13e4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:12:43 +0200 Subject: [PATCH 0248/2548] Remove redundant resolvedRef fields since they're already in SourceInfo --- src/libexpr/primops/flake.cc | 14 ++++++++---- src/libexpr/primops/flake.hh | 10 ++++---- src/nix/flake.cc | 44 ++++++++++++++++++------------------ src/nix/installables.cc | 2 +- 4 files changed, 36 insertions(+), 34 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index d04f665a8..4de742862 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -397,13 +397,17 @@ LockFile entryToLockFile(const LockFile::FlakeEntry & entry) LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) { - LockFile::FlakeEntry entry(resolvedFlake.flake.resolvedRef, resolvedFlake.flake.sourceInfo.narHash); + LockFile::FlakeEntry entry( + resolvedFlake.flake.sourceInfo.resolvedRef, + resolvedFlake.flake.sourceInfo.narHash); for (auto & info : resolvedFlake.flakeDeps) entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { - LockFile::NonFlakeEntry nonEntry(nonFlake.resolvedRef, nonFlake.sourceInfo.narHash); + LockFile::NonFlakeEntry nonEntry( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash); entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); } @@ -540,11 +544,11 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) state.store->isValidPath(path); mkString(*state.allocAttr(v, state.sOutPath), path, {path}); - if (resFlake.flake.resolvedRef.rev) { + if (resFlake.flake.sourceInfo.resolvedRef.rev) { mkString(*state.allocAttr(v, state.symbols.create("rev")), - resFlake.flake.resolvedRef.rev->gitRev()); + resFlake.flake.sourceInfo.resolvedRef.rev->gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), - resFlake.flake.resolvedRef.rev->gitShortRev()); + resFlake.flake.sourceInfo.resolvedRef.rev->gitShortRev()); } if (resFlake.flake.sourceInfo.revCount) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index d90404ae6..a26103736 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -92,7 +92,6 @@ struct Flake { FlakeId id; FlakeRef originalRef; - FlakeRef resolvedRef; std::string description; SourceInfo sourceInfo; std::vector<FlakeRef> requires; @@ -100,18 +99,17 @@ struct Flake Value * vProvides; // FIXME: gc unsigned int epoch; - Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), - resolvedRef(sourceInfo.resolvedRef), sourceInfo(sourceInfo) {}; + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; }; struct NonFlake { FlakeAlias alias; FlakeRef originalRef; - FlakeRef resolvedRef; SourceInfo sourceInfo; - NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : - originalRef(origRef), resolvedRef(sourceInfo.resolvedRef), sourceInfo(sourceInfo) {}; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; }; Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 71a6c16d9..ce5ce8742 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -74,12 +74,12 @@ void printFlakeInfo(const Flake & flake, bool json) { if (json) { nlohmann::json j; j["id"] = flake.id; - j["uri"] = flake.resolvedRef.to_string(); + j["uri"] = flake.sourceInfo.resolvedRef.to_string(); j["description"] = flake.description; - if (flake.resolvedRef.ref) - j["branch"] = *flake.resolvedRef.ref; - if (flake.resolvedRef.rev) - j["revision"] = flake.resolvedRef.rev->to_string(Base16, false); + if (flake.sourceInfo.resolvedRef.ref) + j["branch"] = *flake.sourceInfo.resolvedRef.ref; + if (flake.sourceInfo.resolvedRef.rev) + j["revision"] = flake.sourceInfo.resolvedRef.rev->to_string(Base16, false); if (flake.sourceInfo.revCount) j["revCount"] = *flake.sourceInfo.revCount; j["path"] = flake.sourceInfo.storePath; @@ -87,12 +87,12 @@ void printFlakeInfo(const Flake & flake, bool json) { std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; - std::cout << "URI: " << flake.resolvedRef.to_string() << "\n"; + std::cout << "URI: " << flake.sourceInfo.resolvedRef.to_string() << "\n"; std::cout << "Description: " << flake.description << "\n"; - if (flake.resolvedRef.ref) - std::cout << "Branch: " << *flake.resolvedRef.ref << "\n"; - if (flake.resolvedRef.rev) - std::cout << "Revision: " << flake.resolvedRef.rev->to_string(Base16, false) << "\n"; + if (flake.sourceInfo.resolvedRef.ref) + std::cout << "Branch: " << *flake.sourceInfo.resolvedRef.ref << "\n"; + if (flake.sourceInfo.resolvedRef.rev) + std::cout << "Revision: " << flake.sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; if (flake.sourceInfo.revCount) std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; @@ -104,22 +104,22 @@ void printNonFlakeInfo(const NonFlake & nonFlake, bool json) { if (json) { nlohmann::json j; j["id"] = nonFlake.alias; - j["uri"] = nonFlake.resolvedRef.to_string(); - if (nonFlake.resolvedRef.ref) - j["branch"] = *nonFlake.resolvedRef.ref; - if (nonFlake.resolvedRef.rev) - j["revision"] = nonFlake.resolvedRef.rev->to_string(Base16, false); + j["uri"] = nonFlake.sourceInfo.resolvedRef.to_string(); + if (nonFlake.sourceInfo.resolvedRef.ref) + j["branch"] = *nonFlake.sourceInfo.resolvedRef.ref; + if (nonFlake.sourceInfo.resolvedRef.rev) + j["revision"] = nonFlake.sourceInfo.resolvedRef.rev->to_string(Base16, false); if (nonFlake.sourceInfo.revCount) j["revCount"] = *nonFlake.sourceInfo.revCount; j["path"] = nonFlake.sourceInfo.storePath; std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << nonFlake.alias << "\n"; - std::cout << "URI: " << nonFlake.resolvedRef.to_string() << "\n"; - if (nonFlake.resolvedRef.ref) - std::cout << "Branch: " << *nonFlake.resolvedRef.ref; - if (nonFlake.resolvedRef.rev) - std::cout << "Revision: " << nonFlake.resolvedRef.rev->to_string(Base16, false) << "\n"; + std::cout << "URI: " << nonFlake.sourceInfo.resolvedRef.to_string() << "\n"; + if (nonFlake.sourceInfo.resolvedRef.ref) + std::cout << "Branch: " << *nonFlake.sourceInfo.resolvedRef.ref; + if (nonFlake.sourceInfo.resolvedRef.rev) + std::cout << "Revision: " << nonFlake.sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; if (nonFlake.sourceInfo.revCount) std::cout << "Revcount: " << *nonFlake.sourceInfo.revCount << "\n"; std::cout << "Path: " << nonFlake.sourceInfo.storePath << "\n"; @@ -295,13 +295,13 @@ struct CmdFlakePin : virtual Args, EvalCommand FlakeRegistry userRegistry = *readRegistry(userRegistryPath); auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - it->second = getFlake(*evalState, it->second, true).resolvedRef; + it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; writeRegistry(userRegistry, userRegistryPath); } else { std::shared_ptr<FlakeRegistry> globalReg = evalState->getGlobalFlakeRegistry(); it = globalReg->entries.find(FlakeRef(alias)); if (it != globalReg->entries.end()) { - FlakeRef newRef = getFlake(*evalState, it->second, true).resolvedRef; + auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; userRegistry.entries.insert_or_assign(alias, newRef); writeRegistry(userRegistry, userRegistryPath); } else diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ce09a43d0..4f9161666 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -176,7 +176,7 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const while (!queue.empty()) { const ResolvedFlake & flake = queue.front(); queue.pop(); - if (!std::get_if<FlakeRef::IsPath>(&flake.flake.resolvedRef.data)) + if (!std::get_if<FlakeRef::IsPath>(&flake.flake.sourceInfo.resolvedRef.data)) closure.insert(flake.flake.sourceInfo.storePath); for (const auto & dep : flake.flakeDeps) queue.push(dep.second); From 25e497bf9c3a848afbd7463f132c9229d8a44284 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:14:27 +0200 Subject: [PATCH 0249/2548] nix flake info/deps: Stop progress bar before printing output --- src/nix/flake.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce5ce8742..66fa11e47 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -147,6 +147,8 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON std::queue<ResolvedFlake> todo; todo.push(resolveFlake()); + stopProgressBar(); + while (!todo.empty()) { auto resFlake = std::move(todo.front()); todo.pop(); @@ -204,6 +206,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON void run(nix::ref<nix::Store> store) override { auto flake = getFlake(); + stopProgressBar(); printFlakeInfo(flake, json); } }; From fdf06ce72f9695f57f6215683e4b2e1c6ec463cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:21:06 +0200 Subject: [PATCH 0250/2548] printFlakeInfo/printNonFlakeInfo: Factor out commonality --- src/nix/flake.cc | 66 +++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 66fa11e47..da19caa5a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -70,59 +70,57 @@ struct CmdFlakeList : EvalCommand } }; -void printFlakeInfo(const Flake & flake, bool json) { +static void printSourceInfo(const SourceInfo & sourceInfo) +{ + std::cout << "URI: " << sourceInfo.resolvedRef.to_string() << "\n"; + if (sourceInfo.resolvedRef.ref) + std::cout << "Branch: " << *sourceInfo.resolvedRef.ref; + if (sourceInfo.resolvedRef.rev) + std::cout << "Revision: " << sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; + if (sourceInfo.revCount) + std::cout << "Revcount: " << *sourceInfo.revCount << "\n"; + std::cout << "Path: " << sourceInfo.storePath << "\n"; +} + +static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) +{ + j["uri"] = sourceInfo.resolvedRef.to_string(); + if (sourceInfo.resolvedRef.ref) + j["branch"] = *sourceInfo.resolvedRef.ref; + if (sourceInfo.resolvedRef.rev) + j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false); + if (sourceInfo.revCount) + j["revCount"] = *sourceInfo.revCount; + j["path"] = sourceInfo.storePath; +} + +static void printFlakeInfo(const Flake & flake, bool json) +{ if (json) { nlohmann::json j; j["id"] = flake.id; - j["uri"] = flake.sourceInfo.resolvedRef.to_string(); j["description"] = flake.description; - if (flake.sourceInfo.resolvedRef.ref) - j["branch"] = *flake.sourceInfo.resolvedRef.ref; - if (flake.sourceInfo.resolvedRef.rev) - j["revision"] = flake.sourceInfo.resolvedRef.rev->to_string(Base16, false); - if (flake.sourceInfo.revCount) - j["revCount"] = *flake.sourceInfo.revCount; - j["path"] = flake.sourceInfo.storePath; j["epoch"] = flake.epoch; + sourceInfoToJson(flake.sourceInfo, j); std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << flake.id << "\n"; - std::cout << "URI: " << flake.sourceInfo.resolvedRef.to_string() << "\n"; std::cout << "Description: " << flake.description << "\n"; - if (flake.sourceInfo.resolvedRef.ref) - std::cout << "Branch: " << *flake.sourceInfo.resolvedRef.ref << "\n"; - if (flake.sourceInfo.resolvedRef.rev) - std::cout << "Revision: " << flake.sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; - if (flake.sourceInfo.revCount) - std::cout << "Revcount: " << *flake.sourceInfo.revCount << "\n"; - std::cout << "Path: " << flake.sourceInfo.storePath << "\n"; std::cout << "Epoch: " << flake.epoch << "\n"; + printSourceInfo(flake.sourceInfo); } } -void printNonFlakeInfo(const NonFlake & nonFlake, bool json) { +static void printNonFlakeInfo(const NonFlake & nonFlake, bool json) +{ if (json) { nlohmann::json j; j["id"] = nonFlake.alias; - j["uri"] = nonFlake.sourceInfo.resolvedRef.to_string(); - if (nonFlake.sourceInfo.resolvedRef.ref) - j["branch"] = *nonFlake.sourceInfo.resolvedRef.ref; - if (nonFlake.sourceInfo.resolvedRef.rev) - j["revision"] = nonFlake.sourceInfo.resolvedRef.rev->to_string(Base16, false); - if (nonFlake.sourceInfo.revCount) - j["revCount"] = *nonFlake.sourceInfo.revCount; - j["path"] = nonFlake.sourceInfo.storePath; + printSourceInfo(nonFlake.sourceInfo); std::cout << j.dump(4) << std::endl; } else { std::cout << "ID: " << nonFlake.alias << "\n"; - std::cout << "URI: " << nonFlake.sourceInfo.resolvedRef.to_string() << "\n"; - if (nonFlake.sourceInfo.resolvedRef.ref) - std::cout << "Branch: " << *nonFlake.sourceInfo.resolvedRef.ref; - if (nonFlake.sourceInfo.resolvedRef.rev) - std::cout << "Revision: " << nonFlake.sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; - if (nonFlake.sourceInfo.revCount) - std::cout << "Revcount: " << *nonFlake.sourceInfo.revCount << "\n"; - std::cout << "Path: " << nonFlake.sourceInfo.storePath << "\n"; + printSourceInfo(nonFlake.sourceInfo); } } From e7e7a03baf446bad34939c106aee4b69f5619fd0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 13:22:11 +0200 Subject: [PATCH 0251/2548] nix flake deps: Remove --json flag for now It doesn't produce valid JSON at the moment (but a concatenation of JSON objects). Anyway we probably should merge this command info 'nix flake info'. --- src/nix/flake.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index da19caa5a..810529613 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -125,7 +125,7 @@ static void printNonFlakeInfo(const NonFlake & nonFlake, bool json) } // FIXME: merge info CmdFlakeInfo? -struct CmdFlakeDeps : FlakeCommand, MixJSON +struct CmdFlakeDeps : FlakeCommand { std::string name() override { @@ -152,10 +152,10 @@ struct CmdFlakeDeps : FlakeCommand, MixJSON todo.pop(); for (auto & nonFlake : resFlake.nonFlakeDeps) - printNonFlakeInfo(nonFlake, json); + printNonFlakeInfo(nonFlake, false); for (auto & info : resFlake.flakeDeps) { - printFlakeInfo(info.second.flake, json); + printFlakeInfo(info.second.flake, false); todo.push(info.second); } } From ecee759b80a5a4cefbf15c3670f28bdebb58c404 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 14:01:08 +0200 Subject: [PATCH 0252/2548] callFlake(): Emit source info attributes for non-flake dependencies --- src/libexpr/primops/flake.cc | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 4de742862..162e5c915 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -514,6 +514,23 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); } +static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) +{ + auto & path = sourceInfo.storePath; + state.store->isValidPath(path); + mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); + + if (sourceInfo.resolvedRef.rev) { + mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), + sourceInfo.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), + sourceInfo.resolvedRef.rev->gitShortRev()); + } + + if (sourceInfo.revCount) + mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); +} + void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) { // Construct the resulting attrset '{description, provides, @@ -529,30 +546,18 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { auto vNonFlake = state.allocAttr(v, nonFlake.alias); - state.mkAttrs(*vNonFlake, 4); + state.mkAttrs(*vNonFlake, 8); state.store->isValidPath(nonFlake.sourceInfo.storePath); mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - // FIXME: add rev, shortRev, revCount, ... + emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); } mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); - auto & path = resFlake.flake.sourceInfo.storePath; - state.store->isValidPath(path); - mkString(*state.allocAttr(v, state.sOutPath), path, {path}); - - if (resFlake.flake.sourceInfo.resolvedRef.rev) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), - resFlake.flake.sourceInfo.resolvedRef.rev->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), - resFlake.flake.sourceInfo.resolvedRef.rev->gitShortRev()); - } - - if (resFlake.flake.sourceInfo.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *resFlake.flake.sourceInfo.revCount); + emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v); auto vProvides = state.allocAttr(v, state.symbols.create("provides")); mkApp(*vProvides, *resFlake.flake.vProvides, v); From 46294d60cd46564ea6b40cf9c32759021f9d1fc9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 14:01:57 +0200 Subject: [PATCH 0253/2548] printFlakeInfo: Separate JSON output --- src/nix/flake.cc | 59 ++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 810529613..0fea1993c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -94,34 +94,36 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) j["path"] = sourceInfo.storePath; } -static void printFlakeInfo(const Flake & flake, bool json) +static void printFlakeInfo(const Flake & flake) { - if (json) { - nlohmann::json j; - j["id"] = flake.id; - j["description"] = flake.description; - j["epoch"] = flake.epoch; - sourceInfoToJson(flake.sourceInfo, j); - std::cout << j.dump(4) << std::endl; - } else { - std::cout << "ID: " << flake.id << "\n"; - std::cout << "Description: " << flake.description << "\n"; - std::cout << "Epoch: " << flake.epoch << "\n"; - printSourceInfo(flake.sourceInfo); - } + std::cout << "ID: " << flake.id << "\n"; + std::cout << "Description: " << flake.description << "\n"; + std::cout << "Epoch: " << flake.epoch << "\n"; + printSourceInfo(flake.sourceInfo); } -static void printNonFlakeInfo(const NonFlake & nonFlake, bool json) +static nlohmann::json flakeToJson(const Flake & flake) { - if (json) { - nlohmann::json j; - j["id"] = nonFlake.alias; - printSourceInfo(nonFlake.sourceInfo); - std::cout << j.dump(4) << std::endl; - } else { - std::cout << "ID: " << nonFlake.alias << "\n"; - printSourceInfo(nonFlake.sourceInfo); - } + nlohmann::json j; + j["id"] = flake.id; + j["description"] = flake.description; + j["epoch"] = flake.epoch; + sourceInfoToJson(flake.sourceInfo, j); + return j; +} + +static void printNonFlakeInfo(const NonFlake & nonFlake) +{ + std::cout << "ID: " << nonFlake.alias << "\n"; + printSourceInfo(nonFlake.sourceInfo); +} + +static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) +{ + nlohmann::json j; + j["id"] = nonFlake.alias; + sourceInfoToJson(nonFlake.sourceInfo, j); + return j; } // FIXME: merge info CmdFlakeInfo? @@ -152,10 +154,10 @@ struct CmdFlakeDeps : FlakeCommand todo.pop(); for (auto & nonFlake : resFlake.nonFlakeDeps) - printNonFlakeInfo(nonFlake, false); + printNonFlakeInfo(nonFlake); for (auto & info : resFlake.flakeDeps) { - printFlakeInfo(info.second.flake, false); + printFlakeInfo(info.second.flake); todo.push(info.second); } } @@ -205,7 +207,10 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON { auto flake = getFlake(); stopProgressBar(); - printFlakeInfo(flake, json); + if (json) + std::cout << flakeToJson(flake).dump() << std::endl; + else + printFlakeInfo(flake); } }; From 444786e6d39209d3f0c222af0fbc80846ac7337d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 14:11:19 +0200 Subject: [PATCH 0254/2548] nix flake info: Add missing newline --- src/nix/flake.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0fea1993c..d8c422d3d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -72,14 +72,14 @@ struct CmdFlakeList : EvalCommand static void printSourceInfo(const SourceInfo & sourceInfo) { - std::cout << "URI: " << sourceInfo.resolvedRef.to_string() << "\n"; + std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); if (sourceInfo.resolvedRef.ref) - std::cout << "Branch: " << *sourceInfo.resolvedRef.ref; + std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); if (sourceInfo.resolvedRef.rev) - std::cout << "Revision: " << sourceInfo.resolvedRef.rev->to_string(Base16, false) << "\n"; + std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); if (sourceInfo.revCount) - std::cout << "Revcount: " << *sourceInfo.revCount << "\n"; - std::cout << "Path: " << sourceInfo.storePath << "\n"; + std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount); + std::cout << fmt("Path: %s\n", sourceInfo.storePath); } static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) @@ -96,9 +96,9 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) static void printFlakeInfo(const Flake & flake) { - std::cout << "ID: " << flake.id << "\n"; - std::cout << "Description: " << flake.description << "\n"; - std::cout << "Epoch: " << flake.epoch << "\n"; + std::cout << fmt("ID: %s\n", flake.id); + std::cout << fmt("Description: %s\n", flake.description); + std::cout << fmt("Epoch: %s\n", flake.epoch); printSourceInfo(flake.sourceInfo); } @@ -114,7 +114,7 @@ static nlohmann::json flakeToJson(const Flake & flake) static void printNonFlakeInfo(const NonFlake & nonFlake) { - std::cout << "ID: " << nonFlake.alias << "\n"; + std::cout << fmt("ID: %s\n", nonFlake.alias); printSourceInfo(nonFlake.sourceInfo); } From 22f2744afdfa048de7bd078ba9d24dafbcc7d7c0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 23:05:08 +0200 Subject: [PATCH 0255/2548] Iterate over references --- src/libexpr/parser.y | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 97e1a6584..78a503907 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -83,7 +83,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, assert(!attrPath.empty()); // Checking attrPath validity. // =========================== - for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { @@ -106,18 +106,18 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, } // Expr insertion. // ========================== - if (i->symbol.set()) { + if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { // This attr path is already defined. However, if both // e and the expr pointed by the attr path are two attribute sets, // we want to merge them. // Otherwise, throw an error. - ExprAttrs* ae = dynamic_cast<ExprAttrs *>(e); - ExprAttrs* jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); + auto ae = dynamic_cast<ExprAttrs *>(e); + auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); if (jAttrs && ae) { - for (auto ad: ae->attrs) { - ExprAttrs::AttrDefs::iterator j2 = jAttrs->attrs.find(ad.first); + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs[ad.first] = ad.second; From 0f840483c731f48983832f7f627909f8463f05f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 20:34:02 +0200 Subject: [PATCH 0256/2548] Add date of last commit to SourceInfo This is primarily useful for version string generation, where we need a monotonically increasing number. The revcount is the preferred thing to use, but isn't available for GitHub flakes (since it requires fetching the entire history). The last commit timestamp OTOH can be extracted from GitHub tarballs. --- release.nix | 3 ++- src/libexpr/primops/fetchGit.cc | 9 +++++++- src/libexpr/primops/fetchGit.hh | 1 + src/libexpr/primops/flake.cc | 39 ++++++++++++++++++--------------- src/libexpr/primops/flake.hh | 14 +++++++++++- src/nix/flake.cc | 24 ++++++++++++-------- tests/flakes.sh | 1 + 7 files changed, 61 insertions(+), 30 deletions(-) diff --git a/release.nix b/release.nix index f98e6d6ed..d28c44910 100644 --- a/release.nix +++ b/release.nix @@ -19,7 +19,8 @@ let releaseTools.sourceTarball { name = "nix-tarball"; version = builtins.readFile ./.version; - versionSuffix = if officialRelease then "" else "pre${toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; + versionSuffix = if officialRelease then "" else + "pre${if nix ? lastModified then builtins.substring 0 8 nix.lastModified else toString nix.revCount or 0}_${nix.shortRev or "0000000"}"; src = nix; inherit officialRelease; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f6b096c4a..10f6b6f72 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -69,6 +69,9 @@ GitInfo exportGit(ref<Store> store, std::string uri, gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" })); return gitInfo; } @@ -85,8 +88,9 @@ GitInfo exportGit(ref<Store> store, std::string uri, } deletePath(getCacheDir() + "/nix/git"); + deletePath(getCacheDir() + "/nix/gitv2"); - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false); Path repoDir; if (isLocal) { @@ -181,6 +185,7 @@ GitInfo exportGit(ref<Store> store, std::string uri, if (store->isValidPath(storePath)) { gitInfo.storePath = storePath; gitInfo.revCount = json["revCount"]; + gitInfo.lastModified = json["lastModified"]; return gitInfo; } @@ -200,6 +205,7 @@ GitInfo exportGit(ref<Store> store, std::string uri, gitInfo.storePath = store->addToStore(name, tmpDir); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); + gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() })); nlohmann::json json; json["storePath"] = gitInfo.storePath; @@ -207,6 +213,7 @@ GitInfo exportGit(ref<Store> store, std::string uri, json["name"] = name; json["rev"] = gitInfo.rev.gitRev(); json["revCount"] = gitInfo.revCount; + json["lastModified"] = gitInfo.lastModified; writeFile(storeLink, json.dump()); diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 2ad6a5e5c..006fa8b5f 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -12,6 +12,7 @@ struct GitInfo std::string ref; Hash rev{htSHA1}; uint64_t revCount; + time_t lastModified; }; GitInfo exportGit(ref<Store> store, std::string uri, diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 162e5c915..7cbbf9e99 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -8,6 +8,8 @@ #include <iostream> #include <queue> #include <regex> +#include <ctime> +#include <iomanip> #include <nlohmann/json.hpp> namespace nix { @@ -232,6 +234,18 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { @@ -270,29 +284,13 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool // This downloads the entire git history else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { - auto gitInfo = exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source"); - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - return info; + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); } else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); - auto gitInfo = exportGit(state.store, refData->path, {}, {}, "source"); - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - return info; + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); } else abort(); @@ -529,6 +527,11 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo if (sourceInfo.revCount) mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); } void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index a26103736..0e2706e32 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -81,10 +81,22 @@ void writeRegistry(const FlakeRegistry &, const Path &); struct SourceInfo { + // Immutable flakeref that this source tree was obtained from. FlakeRef resolvedRef; + Path storePath; + + // Number of ancestors of the most recent commit. std::optional<uint64_t> revCount; - Hash narHash; // store path hash + + // NAR hash of the store path. + Hash narHash; + + // A stable timestamp of this source tree. For Git and GitHub + // flakes, the commit date (not author date!) of the most recent + // commit. + std::optional<time_t> lastModified; + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d8c422d3d..7836f0cfe 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include <nlohmann/json.hpp> #include <queue> +#include <iomanip> using namespace nix; @@ -72,14 +73,17 @@ struct CmdFlakeList : EvalCommand static void printSourceInfo(const SourceInfo & sourceInfo) { - std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); + std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); if (sourceInfo.resolvedRef.ref) - std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); + std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); if (sourceInfo.resolvedRef.rev) - std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); + std::cout << fmt("Revision: %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false)); if (sourceInfo.revCount) - std::cout << fmt("Revcount: %s\n", *sourceInfo.revCount); - std::cout << fmt("Path: %s\n", sourceInfo.storePath); + std::cout << fmt("Revisions: %s\n", *sourceInfo.revCount); + if (sourceInfo.lastModified) + std::cout << fmt("Last modified: %s\n", + std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T")); + std::cout << fmt("Path: %s\n", sourceInfo.storePath); } static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) @@ -91,14 +95,16 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false); if (sourceInfo.revCount) j["revCount"] = *sourceInfo.revCount; + if (sourceInfo.lastModified) + j["lastModified"] = *sourceInfo.lastModified; j["path"] = sourceInfo.storePath; } static void printFlakeInfo(const Flake & flake) { - std::cout << fmt("ID: %s\n", flake.id); - std::cout << fmt("Description: %s\n", flake.description); - std::cout << fmt("Epoch: %s\n", flake.epoch); + std::cout << fmt("ID: %s\n", flake.id); + std::cout << fmt("Description: %s\n", flake.description); + std::cout << fmt("Epoch: %s\n", flake.epoch); printSourceInfo(flake.sourceInfo); } @@ -114,7 +120,7 @@ static nlohmann::json flakeToJson(const Flake & flake) static void printNonFlakeInfo(const NonFlake & nonFlake) { - std::cout << fmt("ID: %s\n", nonFlake.alias); + std::cout << fmt("ID: %s\n", nonFlake.alias); printSourceInfo(nonFlake.sourceInfo); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 6081e8939..d95d34c76 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -124,6 +124,7 @@ nix flake info --flake-registry $registry $flake1Dir | grep -q 'ID: *flake1' json=$(nix flake info --flake-registry $registry flake1 --json | jq .) [[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] [[ -d $(echo "$json" | jq -r .path) ]] +[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] # Test 'nix build' on a flake. nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo From ae7b56cd9a5ed8810828736fbb930a7c14ea44ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 28 May 2019 22:35:41 +0200 Subject: [PATCH 0257/2548] Get last commit time of github flakes --- src/libexpr/primops/flake.cc | 2 ++ src/libstore/download.cc | 17 ++++++++++++++--- src/libstore/download.hh | 2 ++ src/libutil/util.cc | 18 +++++++++++++++--- src/libutil/util.hh | 6 ++++-- 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 7cbbf9e99..257b81887 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -265,6 +265,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool request.unpack = true; request.name = "source"; request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; auto result = getDownloader()->downloadCached(state.store, request); if (!result.etag) @@ -278,6 +279,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool SourceInfo info(ref); info.storePath = result.storePath; info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; return info; } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0d1974d3b..0338727c1 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -808,6 +808,7 @@ CachedDownloadResult Downloader::downloadCached( CachedDownloadResult result; result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); + assert(!request.getLastModified); // FIXME return result; } } @@ -892,16 +893,26 @@ CachedDownloadResult Downloader::downloadCached( store->addTempRoot(unpackedStorePath); if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; + else + result.lastModified = lstat(unpackedLink).st_mtime; } if (unpackedStorePath.empty()) { printInfo(format("unpacking '%1%'...") % url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); // FIXME: this requires GNU tar for decompression. - runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); - unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir}); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + result.lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); } - replaceSymlink(unpackedStorePath, unpackedLink); + // Store the last-modified date of the tarball in the symlink + // mtime. This saves us from having to store it somewhere + // else. + replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); storePath = unpackedStorePath; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 404e51195..43b1c5c09 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -49,6 +49,7 @@ struct CachedDownloadRequest Hash expectedHash; unsigned int ttl = settings.tarballTtl; bool gcRoot = false; + bool getLastModified = false; CachedDownloadRequest(const std::string & uri) : uri(uri) { } @@ -62,6 +63,7 @@ struct CachedDownloadResult Path path; std::optional<std::string> etag; std::string effectiveUri; + std::optional<time_t> lastModified; }; class Store; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f82f902fc..92c8957ff 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -22,6 +22,7 @@ #include <sys/ioctl.h> #include <sys/types.h> #include <sys/wait.h> +#include <sys/time.h> #include <unistd.h> #ifdef __APPLE__ @@ -552,20 +553,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 35f9169f6..e05ef1e7d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -142,10 +142,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Wrappers arount read()/write() that read/write exactly the From abdedcdb387628aaa4d9dc362b7e588bbc62baae Mon Sep 17 00:00:00 2001 From: Ding Xiang Fei <dingxiangfei2009@gmail.com> Date: Wed, 29 May 2019 17:01:39 +0800 Subject: [PATCH 0258/2548] bump cpptoml to v0.1.1 --- src/cpptoml/cpptoml.h | 573 +++++++++++++++++++++++++++++------------- 1 file changed, 392 insertions(+), 181 deletions(-) diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h index 07de010b1..5a00da3b4 100644 --- a/src/cpptoml/cpptoml.h +++ b/src/cpptoml/cpptoml.h @@ -4,8 +4,8 @@ * @date May 2013 */ -#ifndef _CPPTOML_H_ -#define _CPPTOML_H_ +#ifndef CPPTOML_H +#define CPPTOML_H #include <algorithm> #include <cassert> @@ -84,11 +84,12 @@ class option return &value_; } - const T& value_or(const T& alternative) const + template <class U> + T value_or(U&& alternative) const { if (!empty_) return value_; - return alternative; + return static_cast<T>(std::forward<U>(alternative)); } private: @@ -295,13 +296,12 @@ struct valid_value_or_string_convertible }; template <class T> -struct value_traits<T, typename std:: - enable_if<valid_value_or_string_convertible<T>:: - value>::type> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> { - using value_type = typename std:: - conditional<valid_value<typename std::decay<T>::type>::value, - typename std::decay<T>::type, std::string>::type; + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; using type = value<value_type>; @@ -312,12 +312,11 @@ struct value_traits<T, typename std:: }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_floating_point< - typename std::decay<T>::type>::value>:: - type> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> { using value_type = typename std::decay<T>::type; @@ -330,11 +329,11 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_signed<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -356,11 +355,10 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_unsigned<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -395,10 +393,15 @@ struct array_of_trait<array> template <class T> inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); inline std::shared_ptr<array> make_array(); + +namespace detail +{ template <class T> inline std::shared_ptr<T> make_element(); +} + inline std::shared_ptr<table> make_table(); -inline std::shared_ptr<table_array> make_table_array(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); #if defined(CPPTOML_NO_RTTI) /// Base type used to store underlying data type explicitly if RTTI is disabled @@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base> #if defined(CPPTOML_NO_RTTI) base_type type() const { - return type_; + return type_; } protected: @@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as() if (type() == base_type::INT) { auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); - return make_value<double>(static_cast<double>(v->get()));; + return make_value<double>(static_cast<double>(v->get())); } #else if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) @@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const { #if defined(CPPTOML_NO_RTTI) if (type() == base_type::FLOAT) - return std::static_pointer_cast<const value<double>>(shared_from_this()); + return std::static_pointer_cast<const value<double>>( + shared_from_this()); if (type() == base_type::INT) { @@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<array> make_element<array>() { return make_array(); } +} // namespace detail /** * Obtains a option<vector<T>>. The option will be empty if the array @@ -1060,7 +1067,7 @@ class table; class table_array : public base { friend class table; - friend std::shared_ptr<table_array> make_table_array(); + friend std::shared_ptr<table_array> make_table_array(bool); public: std::shared_ptr<base> clone() const override; @@ -1152,14 +1159,25 @@ class table_array : public base array_.reserve(n); } + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + private: #if defined(CPPTOML_NO_RTTI) - table_array() : base(base_type::TABLE_ARRAY) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) { // nothing } #else - table_array() + table_array(bool is_inline = false) : is_inline_(is_inline) { // nothing } @@ -1169,26 +1187,30 @@ class table_array : public base table_array& operator=(const table_array& rhs) = delete; std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; }; -inline std::shared_ptr<table_array> make_table_array() +inline std::shared_ptr<table_array> make_table_array(bool is_inline) { struct make_shared_enabler : public table_array { - make_shared_enabler() + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) { // nothing } }; - return std::make_shared<make_shared_enabler>(); + return std::make_shared<make_shared_enabler>(is_inline); } +namespace detail +{ template <> inline std::shared_ptr<table_array> make_element<table_array>() { - return make_table_array(); + return make_table_array(true); } +} // namespace detail // The below are overloads for fetching specific value types out of a value // where special casting behavior (like bounds checking) is desired @@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<table> make_element<table>() { return make_table(); } +} // namespace detail template <class T> std::shared_ptr<base> value<T>::clone() const @@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const inline std::shared_ptr<base> table_array::clone() const { - auto result = make_table_array(); + auto result = make_table_array(is_inline()); result->reserve(array_.size()); for (const auto& ptr : array_) result->array_.push_back(ptr->clone()->as_table()); @@ -1738,6 +1763,11 @@ inline bool is_number(char c) return c >= '0' && c <= '9'; } +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + /** * Helper object for consuming expected characters. */ @@ -1766,6 +1796,13 @@ class consumer [&](char c) { (*this)(c); }); } + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + int eat_digits(int len) { int val = 0; @@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line) line.push_back(static_cast<char>(c)); } } -} +} // namespace detail /** * The parser class. @@ -1914,21 +1951,25 @@ class parser std::string full_table_name; bool inserted = false; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table name"); if (!full_table_name.empty()) - full_table_name += "."; + full_table_name += '.'; full_table_name += part; if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif if (b->is_table()) curr_table = static_cast<table*>(b.get()); else if (b->is_table_array()) @@ -1946,16 +1987,23 @@ class parser curr_table->insert(part, make_table()); curr_table = static_cast<table*>(curr_table->get(part).get()); } - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); if (it == end) throw_parse_exception( "Unterminated table declaration; did you forget a ']'?"); + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + // table already existed if (!inserted) { @@ -1969,8 +2017,9 @@ class parser // since it has already been defined. If there aren't any // values, then it was implicitly created by something like // [a.b] - if (curr_table->empty() || std::any_of(curr_table->begin(), - curr_table->end(), is_value)) + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) { throw_parse_exception("Redefinition of table " + full_table_name); @@ -1989,36 +2038,45 @@ class parser if (it == end || *it == ']') throw_parse_exception("Table array name cannot be empty"); - std::string full_ta_name; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table array name"); if (!full_ta_name.empty()) - full_ta_name += "."; + full_ta_name += '.'; full_ta_name += part; - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif // if this is the end of the table array name, add an - // element to the table array that we just looked up + // element to the table array that we just looked up, + // provided it was not declared inline if (it != end && *it == ']') { if (!b->is_table_array()) + { throw_parse_exception("Key " + full_ta_name + " is not a table array"); + } + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + v->get().push_back(make_table()); curr_table = v->get().back().get(); } @@ -2059,15 +2117,16 @@ class parser = static_cast<table*>(curr_table->get(part).get()); } } - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); // consume the last "]]" - if (it == end) + auto eat = make_consumer(it, end, [this]() { throw_parse_exception("Unterminated table array name"); - ++it; - if (it == end) - throw_parse_exception("Unterminated table array name"); - ++it; + }); + eat(']'); + eat(']'); consume_whitespace(it, end); eol_or_comment(it, end); @@ -2076,7 +2135,35 @@ class parser void parse_key_value(std::string::iterator& it, std::string::iterator& end, table* curr_table) { - auto key = parse_key(it, end, [](char c) { return c == '='; }); + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + if (curr_table->contains(key)) throw_parse_exception("Key " + key + " already present"); if (it == end || *it != '=') @@ -2087,18 +2174,57 @@ class parser consume_whitespace(it, end); } - template <class Function> - std::string parse_key(std::string::iterator& it, - const std::string::iterator& end, Function&& fun) + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) { consume_whitespace(it, end); - if (*it == '"') + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') { - return parse_quoted_key(it, end); + return string_literal(it, end, *it); } else { - auto bke = std::find_if(it, end, std::forward<Function>(fun)); + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); return parse_bare_key(it, bke); } } @@ -2142,12 +2268,6 @@ class parser return key; } - std::string parse_quoted_key(std::string::iterator& it, - const std::string::iterator& end) - { - return string_literal(it, end, '"'); - } - enum class parse_type { STRING = 1, @@ -2193,7 +2313,7 @@ class parser parse_type determine_value_type(const std::string::iterator& it, const std::string::iterator& end) { - if(it == end) + if (it == end) { throw_parse_exception("Failed to parse value type"); } @@ -2209,7 +2329,11 @@ class parser { return *dtype; } - else if (is_number(*it) || *it == '-' || *it == '+') + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) { return determine_number_type(it, end); } @@ -2235,6 +2359,13 @@ class parser auto check_it = it; if (*check_it == '-' || *check_it == '+') ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + while (check_it != end && is_number(*check_it)) ++check_it; if (check_it != end && *check_it == '.') @@ -2283,57 +2414,56 @@ class parser bool consuming = false; std::shared_ptr<value<std::string>> ret; - auto handle_line - = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); - // whole line is whitespace - if (local_it == local_end) - return; - } + // whole line is whitespace + if (local_it == local_end) + return; + } - consuming = false; + consuming = false; - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } - ss << parse_escape_code(local_it, local_end); - continue; - } + ss << parse_escape_code(local_it, local_end); + continue; + } - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value<std::string>(ss.str()); - break; - } - } + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } - ss << *local_it++; - } - }; + ss << *local_it++; + } + }; // handle the remainder of the current line handle_line(it, end); @@ -2514,17 +2644,13 @@ class parser return value; } - bool is_hex(char c) - { - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - uint32_t hex_to_digit(char c) { if (is_number(c)) return static_cast<uint32_t>(c - '0'); - return 10 + static_cast<uint32_t>( - c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); } std::shared_ptr<base> parse_number(std::string::iterator& it, @@ -2538,25 +2664,6 @@ class parser ++check_it; }; - eat_sign(); - - auto eat_numbers = [&]() { - auto beg = check_it; - while (check_it != end && is_number(*check_it)) - { - ++check_it; - if (check_it != end && *check_it == '_') - { - ++check_it; - if (check_it == end || !is_number(*check_it)) - throw_parse_exception("Malformed number"); - } - } - - if (check_it == beg) - throw_parse_exception("Malformed number"); - }; - auto check_no_leading_zero = [&]() { if (check_it != end && *check_it == '0' && check_it + 1 != check_end && check_it[1] != '.') @@ -2565,7 +2672,80 @@ class parser } }; + auto eat_digits = [&](bool (*check_char)(char)) { + auto beg = check_it; + while (check_it != end && check_char(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !check_char(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') + { + eat_hex(); + return parse_int(it, check_it, 16); + } + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + + eat_sign(); check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + eat_numbers(); if (check_it != end @@ -2604,14 +2784,17 @@ class parser } std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, - const std::string::iterator& end) + const std::string::iterator& end, + int base = 10, + const char* prefix = "") { std::string v{it, end}; + v = prefix + v; v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); it = end; try { - return make_value<int64_t>(std::stoll(v)); + return make_value<int64_t>(std::stoll(v, nullptr, base)); } catch (const std::invalid_argument& ex) { @@ -2674,18 +2857,33 @@ class parser std::string::iterator find_end_of_number(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { + auto ret = std::find_if(it, end, [](char c) { return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+'; + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; } std::string::iterator find_end_of_date(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' - && c != '+' && c != '.'; + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; }); } @@ -2754,7 +2952,7 @@ class parser if (it == date_end) return make_value(ldate); - eat('T'); + eat.eat_or('T', ' '); local_datetime ldt; static_cast<local_date&>(ldt) = ldate; @@ -2850,9 +3048,9 @@ class parser auto arr = make_array(); while (it != end && *it != ']') { - auto value = parse_value(it, end); - if (auto v = value->as<Value>()) - arr->get().push_back(value); + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); else throw_parse_exception("Arrays must be homogeneous"); skip_whitespace_and_comments(it, end); @@ -2871,7 +3069,7 @@ class parser std::string::iterator& it, std::string::iterator& end) { - auto arr = make_element<Object>(); + auto arr = detail::make_element<Object>(); while (it != end && *it != ']') { @@ -2881,7 +3079,7 @@ class parser arr->get().push_back(((*this).*fun)(it, end)); skip_whitespace_and_comments(it, end); - if (*it != ',') + if (it == end || *it != ',') break; ++it; @@ -2906,8 +3104,11 @@ class parser throw_parse_exception("Unterminated inline table"); consume_whitespace(it, end); - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } } while (*it == ','); if (it == end || *it != '}') @@ -2987,7 +3188,8 @@ class parser if (it[4] != '-' || it[7] != '-') return {}; - if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) { // datetime type auto time_end = find_end_of_time(it + 11, date_end); @@ -3243,7 +3445,7 @@ class toml_writer { res += "\\\\"; } - else if ((const uint32_t)*it <= 0x001f) + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) { res += "\\u"; std::stringstream ss; @@ -3274,12 +3476,21 @@ class toml_writer */ void write(const value<double>& v) { - std::ios::fmtflags flags{stream_.flags()}; + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); - stream_ << std::showpoint; - write(v.get()); + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); - stream_.flags(flags); + stream_ << double_str; + has_naked_endline_ = false; } /** @@ -3287,9 +3498,9 @@ class toml_writer * offset_datetime. */ template <class T> - typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, - local_datetime, - offset_datetime>::value>::type + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type write(const value<T>& v) { write(v.get()); @@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a) a.accept(writer); return stream; } -} -#endif +} // namespace cpptoml +#endif // CPPTOML_H From cfd74aef1ef98e8830b18a300057dc8b80df2a9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 12:12:02 +0200 Subject: [PATCH 0259/2548] Fix eval-okay-fromTOML test Turns out we were mis-parsing single-quoted attributes, e.g. 'key2'. --- tests/lang/eval-okay-fromTOML.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 5b9d47122..10ad2fe99 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] From 17ef3e6f411b2cf1fa87593ad84bc2a94a1edd93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 12:22:52 +0200 Subject: [PATCH 0260/2548] Enable more fromTOML tests cpptoml now parses almost all examples from the spec. --- tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 30 ++++++++++++++---------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 10ad2fe99..392ff7a72 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 8e7cbd1c6..5626ef338 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -46,15 +46,15 @@ "character encoding" = "value" "ʎǝʞ" = "value" 'key2' = "value" - #'quoted "value"' = "value" + 'quoted "value"' = "value" name = "Orange" - # FIXME: cpptoml doesn't handle dotted keys properly yet. - #physical.color = "orange" - #physical.shape = "round" - #site."google.com" = true + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + # This is legal according to the spec, but cpptoml doesn't handle it. #a.b.c = 1 #a.d = 2 @@ -68,16 +68,14 @@ int6 = 5_349_221 int7 = 1_2_3_4_5 - # FIXME: cpptoml doesn't support these yet: + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef - #hex1 = 0xDEADBEEF - #hex2 = 0xdeadbeef - #hex3 = 0xdead_beef + oct1 = 0o01234567 + oct2 = 0o755 - #oct1 = 0o01234567 - #oct2 = 0o755 - - #bin1 = 0b11010110 + bin1 = 0b11010110 flt1 = +1.0 flt2 = 3.1415 @@ -126,8 +124,8 @@ key1 = "another string" key2 = 456 - #[dog."tater.man"] - #type.name = "pug" + [dog."tater.man"] + type.name = "pug" [a.b.c] [ d.e.f ] @@ -137,7 +135,7 @@ name = { first = "Tom", last = "Preston-Werner" } point = { x = 1, y = 2 } - #animal = { type.name = "pug" } + animal = { type.name = "pug" } [[products]] name = "Hammer" From c356d034f3ad3639b52792828ced2cd5e26426ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 15:12:22 +0200 Subject: [PATCH 0261/2548] Make unsupported flake attributes a fatal error --- src/libexpr/primops/flake.cc | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 257b81887..0fb562895 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -327,7 +327,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe state.forceAttrs(vInfo); - if (auto epoch = vInfo.attrs->get(state.symbols.create("epoch"))) { + auto sEpoch = state.symbols.create("epoch"); + + if (auto epoch = vInfo.attrs->get(sEpoch)) { flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); if (flake.epoch > 2019) throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); @@ -342,14 +344,18 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe if (auto description = vInfo.attrs->get(state.sDescription)) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); - if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) { + auto sRequires = state.symbols.create("requires"); + + if (auto requires = vInfo.attrs->get(sRequires)) { state.forceList(*(**requires).value, *(**requires).pos); for (unsigned int n = 0; n < (**requires).value->listSize(); ++n) flake.requires.push_back(FlakeRef(state.forceStringNoCtx( *(**requires).value->listElems()[n], *(**requires).pos))); } - if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(state.symbols.create("nonFlakeRequires"))) { + auto sNonFlakeRequires = state.symbols.create("nonFlakeRequires"); + + if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(sNonFlakeRequires)) { state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos); for (Attr attr : *(*(**nonFlakeRequires).value).attrs) { std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); @@ -358,12 +364,25 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } } - if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) { + auto sProvides = state.symbols.create("provides"); + + if (auto provides = vInfo.attrs->get(sProvides)) { state.forceFunction(*(**provides).value, *(**provides).pos); flake.vProvides = (**provides).value; } else throw Error("flake '%s' lacks attribute 'provides'", flakeRef); + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEpoch && + attr.name != state.sName && + attr.name != state.sDescription && + attr.name != sRequires && + attr.name != sNonFlakeRequires && + attr.name != sProvides) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + flakeRef, attr.name, *attr.pos); + } + return flake; } From 6e4a8c47f46041a94135c1e91ce4ab606d05f8ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 15:31:07 +0200 Subject: [PATCH 0262/2548] Put flake-related stuff in its own namespace --- src/libexpr/eval.hh | 9 +++-- src/libexpr/primops/flake.cc | 64 ++++++++++++++++++++---------------- src/libexpr/primops/flake.hh | 10 ++++-- src/nix/command.hh | 5 ++- src/nix/flake.cc | 5 +-- src/nix/installables.cc | 11 ++++--- 6 files changed, 62 insertions(+), 42 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1e45bc1a8..46c6ea271 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -17,7 +17,10 @@ namespace nix { class Store; class EvalState; enum RepairFlag : bool; + +namespace flake { struct FlakeRegistry; +} typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -323,12 +326,12 @@ private: public: - const std::vector<std::shared_ptr<FlakeRegistry>> getFlakeRegistries(); + const std::vector<std::shared_ptr<flake::FlakeRegistry>> getFlakeRegistries(); - std::shared_ptr<FlakeRegistry> getGlobalFlakeRegistry(); + std::shared_ptr<flake::FlakeRegistry> getGlobalFlakeRegistry(); private: - std::shared_ptr<FlakeRegistry> _globalFlakeRegistry; + std::shared_ptr<flake::FlakeRegistry> _globalFlakeRegistry; std::once_flag _globalFlakeRegistryInit; }; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 0fb562895..5223b3df4 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -14,6 +14,10 @@ namespace nix { +using namespace flake; + +namespace flake { + /* Read a registry. */ std::shared_ptr<FlakeRegistry> readRegistry(const Path & path) { @@ -133,24 +137,6 @@ void writeLockFile(const LockFile & lockFile, const Path & path) writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file } -std::shared_ptr<FlakeRegistry> EvalState::getGlobalFlakeRegistry() -{ - std::call_once(_globalFlakeRegistryInit, [&]() { - auto path = evalSettings.flakeRegistry; - - if (!hasPrefix(path, "/")) { - CachedDownloadRequest request(evalSettings.flakeRegistry); - request.name = "flake-registry.json"; - request.gcRoot = true; - path = getDownloader()->downloadCached(store, request).path; - } - - _globalFlakeRegistry = readRegistry(path); - }); - - return _globalFlakeRegistry; -} - Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -170,17 +156,6 @@ std::shared_ptr<FlakeRegistry> getFlagRegistry(RegistryOverrides registryOverrid return flagRegistry; } -// This always returns a vector with flakeReg, userReg, globalReg. -// If one of them doesn't exist, the registry is left empty but does exist. -const Registries EvalState::getFlakeRegistries() -{ - Registries registries; - registries.push_back(getFlagRegistry(registryOverrides)); - registries.push_back(getUserRegistry()); - registries.push_back(getGlobalFlakeRegistry()); - return registries; -} - static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, std::vector<FlakeRef> pastSearches = {}); @@ -637,3 +612,34 @@ void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, } } + +std::shared_ptr<flake::FlakeRegistry> EvalState::getGlobalFlakeRegistry() +{ + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; +} + +// This always returns a vector with flakeReg, userReg, globalReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() +{ + Registries registries; + registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalFlakeRegistry()); + return registries; +} + +} diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 0e2706e32..309bf6db5 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -5,13 +5,15 @@ namespace nix { +struct Value; +class EvalState; + +namespace flake { + static const size_t FLAG_REGISTRY = 0; static const size_t USER_REGISTRY = 1; static const size_t GLOBAL_REGISTRY = 2; -struct Value; -class EvalState; - struct FlakeRegistry { std::map<FlakeRef, FlakeRef> entries; @@ -143,3 +145,5 @@ void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFil void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); } + +} diff --git a/src/nix/command.hh b/src/nix/command.hh index a841b879a..26c308331 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -12,7 +12,10 @@ struct Value; class Bindings; class EvalState; class Store; + +namespace flake { enum HandleLockFile : unsigned int; +} /* A command that require a Nix store. */ struct StoreCommand : virtual Command @@ -71,7 +74,7 @@ struct MixFlakeOptions : virtual Args MixFlakeOptions(); - HandleLockFile getLockFileMode(); + flake::HandleLockFile getLockFileMode(); }; struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7836f0cfe..c2781531d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -10,6 +10,7 @@ #include <iomanip> using namespace nix; +using namespace nix::flake; class FlakeCommand : virtual Args, public EvalCommand, public MixFlakeOptions { @@ -33,12 +34,12 @@ public: Flake getFlake() { auto evalState = getEvalState(); - return nix::getFlake(*evalState, getFlakeRef(), useRegistries); + return flake::getFlake(*evalState, getFlakeRef(), useRegistries); } ResolvedFlake resolveFlake() { - return nix::resolveFlake(*getEvalState(), getFlakeRef(), getLockFileMode()); + return flake::resolveFlake(*getEvalState(), getFlakeRef(), getLockFileMode()); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 4f9161666..df5214f13 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -32,8 +32,9 @@ MixFlakeOptions::MixFlakeOptions() .set(&useRegistries, false); } -HandleLockFile MixFlakeOptions::getLockFileMode() +flake::HandleLockFile MixFlakeOptions::getLockFileMode() { + using namespace flake; return useRegistries ? recreateLockFile @@ -163,18 +164,20 @@ struct InstallableAttrPath : InstallableValue } }; -void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const ResolvedFlake & resFlake) +void makeFlakeClosureGCRoot(Store & store, + const FlakeRef & origFlakeRef, + const flake::ResolvedFlake & resFlake) { if (std::get_if<FlakeRef::IsPath>(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ PathSet closure; - std::queue<std::reference_wrapper<const ResolvedFlake>> queue; + std::queue<std::reference_wrapper<const flake::ResolvedFlake>> queue; queue.push(resFlake); while (!queue.empty()) { - const ResolvedFlake & flake = queue.front(); + const flake::ResolvedFlake & flake = queue.front(); queue.pop(); if (!std::get_if<FlakeRef::IsPath>(&flake.flake.sourceInfo.resolvedRef.data)) closure.insert(flake.flake.sourceInfo.storePath); From 6ae4437acbd5967cf4211ce8c8685a6f1273aaeb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 15:44:48 +0200 Subject: [PATCH 0263/2548] Remove makeFlakeValue() --- src/libexpr/primops/flake.cc | 11 ++--------- src/libexpr/primops/flake.hh | 2 -- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 5223b3df4..0d762f2a1 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -566,18 +566,11 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) v.attrs->sort(); } -// Return the `provides` of the top flake, while assigning to `v` the provides -// of the dependencies as well. -void makeFlakeValue(EvalState & state, const FlakeRef & flakeRef, HandleLockFile handle, Value & v) -{ - callFlake(state, resolveFlake(state, flakeRef, handle), v); -} - // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - makeFlakeValue(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? AllPure : UseUpdatedLockFile, v); + callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); } static RegisterPrimOp r2("getFlake", 1, prim_getFlake); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 309bf6db5..340b97c65 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -75,8 +75,6 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -void makeFlakeValue(EvalState &, const FlakeRef &, HandleLockFile, Value &); - std::shared_ptr<FlakeRegistry> readRegistry(const Path &); void writeRegistry(const FlakeRegistry &, const Path &); From a8251ba2ed9e833ef3e8b20d233064aba74446db Mon Sep 17 00:00:00 2001 From: Johannes Climacus <johannesclimacus@tuta.io> Date: Sat, 25 May 2019 05:06:43 -0400 Subject: [PATCH 0264/2548] Replace `type` with `command -v` in install script In POSIX sh, `type` is undefined. cf. https://pubs.opengroup.org/onlinepubs/9699919799/utilities/command.html#tag_20_22_04 --- scripts/install.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/install.in b/scripts/install.in index 7bff7b216..4857638c0 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -18,7 +18,7 @@ cleanup() { trap cleanup EXIT INT QUIT TERM require_util() { - type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + command -v "$1" > /dev/null 2>&1 || oops "you do not have '$1' installed, which I need to $2" } @@ -41,11 +41,11 @@ require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" -if type sha256sum > /dev/null 2>&1; then +if command -v sha256sum > /dev/null 2>&1; then hash2="$(sha256sum -b "$tarball" | cut -c1-64)" -elif type shasum > /dev/null 2>&1; then +elif command -v shasum > /dev/null 2>&1; then hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" -elif type openssl > /dev/null 2>&1; then +elif command -v openssl > /dev/null 2>&1; then hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" else oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" From de00ed15d3fd70ffd897d660fc337029ab50a0bf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 16:30:00 +0200 Subject: [PATCH 0265/2548] Doh --- shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 228a6685b..d7e63bad3 100644 --- a/shell.nix +++ b/shell.nix @@ -2,7 +2,7 @@ , nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz }: -with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) { system = builtins.currentSystem or "x86_64-linux"; }; +with import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; with import ./release-common.nix { inherit pkgs; }; From e0aaf05f4fde3096a86c6481ada64aae53bfce93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 17:25:41 +0200 Subject: [PATCH 0266/2548] Add 'nix flake check' command This evaluates all the 'provides' of a flake and builds the 'check' attributes. --- flake.nix | 8 +++-- src/nix/flake.cc | 93 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index ab316c7c6..dd7d7992c 100644 --- a/flake.nix +++ b/flake.nix @@ -14,13 +14,17 @@ nixpkgs = deps.nixpkgs; }; - packages.nix = hydraJobs.build.x86_64-linux; + checks.binaryTarball = hydraJobs.binaryTarball.x86_64-linux; + + packages = { + nix = hydraJobs.build.x86_64-linux; + nix-perl-bindings = hydraJobs.perlBindings.x86_64-linux; + }; defaultPackage = packages.nix; devShell = import ./shell.nix { nixpkgs = deps.nixpkgs; }; - }; } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index c2781531d..f0231e263 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -3,7 +3,10 @@ #include "shared.hh" #include "progress-bar.hh" #include "eval.hh" +#include "eval-inline.hh" #include "primops/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" #include <nlohmann/json.hpp> #include <queue> @@ -208,8 +211,6 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON return "list info about a given flake"; } - CmdFlakeInfo () { } - void run(nix::ref<nix::Store> store) override { auto flake = getFlake(); @@ -221,6 +222,93 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON } }; +static void enumerateProvides(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide)> callback) +{ + state.forceAttrs(vFlake); + + auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; + + state.forceAttrs(*vProvides); + + for (auto & attr : *vProvides->attrs) + callback(attr.name, *attr.value); +} + +struct CmdFlakeCheck : FlakeCommand, MixJSON +{ + bool build = true; + + CmdFlakeCheck() + { + mkFlag() + .longName("no-build") + .description("do not build checks") + .set(&build, false); + } + + std::string name() override + { + return "check"; + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + void run(nix::ref<nix::Store> store) override + { + auto state = getEvalState(); + auto flake = resolveFlake(); + + PathSet drvPaths; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateProvides(*state, + *vFlake, + [&](const std::string & name, Value & vProvide) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vProvide); + + if (name == "checks") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) { + try { + auto drvInfo = getDerivation(*state, *aCheck.value, false); + if (!drvInfo) + throw Error("flake output 'check.%s' is not a derivation", aCheck.name); + drvPaths.insert(drvInfo->queryDrvPath()); + // FIXME: check meta attributes? + } catch (Error & e) { + e.addPrefix(fmt("while checking flake check '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", aCheck.name)); + throw; + } + } + } + + } catch (Error & e) { + e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); + throw; + } + }); + } + + if (build) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + } +}; + struct CmdFlakeAdd : MixEvalArgs, Command { FlakeUri alias; @@ -387,6 +475,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command : MultiCommand({make_ref<CmdFlakeList>() , make_ref<CmdFlakeUpdate>() , make_ref<CmdFlakeInfo>() + , make_ref<CmdFlakeCheck>() , make_ref<CmdFlakeDeps>() , make_ref<CmdFlakeAdd>() , make_ref<CmdFlakeRemove>() From 0e32b32fa3655e8a3e028ebb5ac838164a606132 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 20:57:08 +0200 Subject: [PATCH 0267/2548] nix flake check: Check defaultPackage, devShell and packages --- src/nix/flake.cc | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index f0231e263..19e97aed9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -262,6 +262,19 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = resolveFlake(); + auto checkDerivation = [&](const std::string & attrPath, Value & v) { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return drvInfo->queryDrvPath(); + } catch (Error & e) { + e.addPrefix(fmt("while checking flake attribute '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + throw; + } + }; + PathSet drvPaths; { @@ -281,20 +294,21 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON if (name == "checks") { state->forceAttrs(vProvide); - for (auto & aCheck : *vProvide.attrs) { - try { - auto drvInfo = getDerivation(*state, *aCheck.value, false); - if (!drvInfo) - throw Error("flake output 'check.%s' is not a derivation", aCheck.name); - drvPaths.insert(drvInfo->queryDrvPath()); - // FIXME: check meta attributes? - } catch (Error & e) { - e.addPrefix(fmt("while checking flake check '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", aCheck.name)); - throw; - } - } + for (auto & aCheck : *vProvide.attrs) + drvPaths.insert(checkDerivation( + name + "." + (std::string) aCheck.name, *aCheck.value)); } + else if (name == "packages") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + checkDerivation( + name + "." + (std::string) aCheck.name, *aCheck.value); + } + + else if (name == "defaultPackage" || name == "devShell") + checkDerivation(name, vProvide); + } catch (Error & e) { e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); throw; From a9d3524e1f4f0212184d611b3ff3b520619dff8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 21:00:44 +0200 Subject: [PATCH 0268/2548] nix flake check: Use read-only mode if we're not building --- src/nix/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 19e97aed9..0be003da2 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -259,6 +259,8 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON void run(nix::ref<nix::Store> store) override { + settings.readOnlyMode = !build; + auto state = getEvalState(); auto flake = resolveFlake(); From b70fc8f30c66df9c5ba4be06e4a808df1bba7d3a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 21:00:58 +0200 Subject: [PATCH 0269/2548] flake.nix: Add more checks --- flake.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index dd7d7992c..20ad2bc7e 100644 --- a/flake.nix +++ b/flake.nix @@ -14,7 +14,12 @@ nixpkgs = deps.nixpkgs; }; - checks.binaryTarball = hydraJobs.binaryTarball.x86_64-linux; + checks = { + binaryTarball = hydraJobs.binaryTarball.x86_64-linux; + perlBindings = hydraJobs.perlBindings.x86_64-linux; + inherit (hydraJobs.tests) remoteBuilds nix-copy-closure; + setuid = hydraJobs.tests.setuid.x86_64-linux; + }; packages = { nix = hydraJobs.build.x86_64-linux; From 3488fa7c6cef487d3f9501e89894f9e632e678db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 21:30:22 +0200 Subject: [PATCH 0270/2548] Hack: Use legacyPackages from Nixpkgs Nixpkgs doesn't provide a clean "packages" set yet, so until that's the case, look for packages in "legacyPackages" as well. --- src/nix/installables.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index df5214f13..ea12cd79c 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -236,9 +236,9 @@ struct InstallableFlake : InstallableValue auto emptyArgs = state.allocBindings(0); - // As a convenience, look for the attribute in - // 'provides.packages'. if (searchPackages) { + // As a convenience, look for the attribute in + // 'provides.packages'. if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { try { auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); @@ -247,6 +247,17 @@ struct InstallableFlake : InstallableValue } catch (AttrPathNotFound & e) { } } + + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + if (auto aPackages = *vProvides->attrs->get(state.symbols.create("legacyPackages"))) { + try { + auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } } // Otherwise, look for it in 'provides'. From 49436bdbb77f32ffec2035e836add04f98be49e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 22:17:08 +0200 Subject: [PATCH 0271/2548] nix flake info --json: List the "provides" It also lists the contents of "checks" and "packages". For example: $ nix flake info --json | jq { "branch": "HEAD", "description": "The purely functional package manager", "epoch": 2019, "id": "nix", "lastModified": 1559161142, "path": "/nix/store/2w2qla8735dbxah8gai8r1nsbf5x4f5d-source", "provides": { "checks": { "binaryTarball": {}, "nix-copy-closure": {}, "perlBindings": {}, "remoteBuilds": {}, "setuid": {} }, "defaultPackage": {}, "devShell": {}, "hydraJobs": {}, "packages": { "nix": {}, "nix-perl-bindings": {} } }, "revCount": 6955, "revision": "8cb24e04e8b6cc60e2504733afe78e0eadafcd98", "uri": "/home/eelco/Dev/nix" } Fixes #2820. --- src/nix/flake.cc | 58 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0be003da2..872ec2849 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -199,6 +199,19 @@ struct CmdFlakeUpdate : FlakeCommand } }; +static void enumerateProvides(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide)> callback) +{ + state.forceAttrs(vFlake); + + auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; + + state.forceAttrs(*vProvides); + + for (auto & attr : *vProvides->attrs) + callback(attr.name, *attr.value); +} + struct CmdFlakeInfo : FlakeCommand, MixJSON { std::string name() override @@ -215,26 +228,39 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON { auto flake = getFlake(); stopProgressBar(); - if (json) - std::cout << flakeToJson(flake).dump() << std::endl; - else + + if (json) { + auto json = flakeToJson(flake); + + auto state = getEvalState(); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + auto provides = nlohmann::json::object(); + + enumerateProvides(*state, + *vFlake, + [&](const std::string & name, Value & vProvide) { + auto provide = nlohmann::json::object(); + + if (name == "checks" || name == "packages") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + provide[aCheck.name] = nlohmann::json::object(); + } + + provides[name] = provide; + }); + + json["provides"] = std::move(provides); + + std::cout << json.dump() << std::endl; + } else printFlakeInfo(flake); } }; -static void enumerateProvides(EvalState & state, Value & vFlake, - std::function<void(const std::string & name, Value & vProvide)> callback) -{ - state.forceAttrs(vFlake); - - auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; - - state.forceAttrs(*vProvides); - - for (auto & attr : *vProvides->attrs) - callback(attr.name, *attr.value); -} - struct CmdFlakeCheck : FlakeCommand, MixJSON { bool build = true; From 3b1cc8b0cbc0a2530e4671231257a497db8eede2 Mon Sep 17 00:00:00 2001 From: ln-nl <44435420+ln-nl@users.noreply.github.com> Date: Thu, 30 May 2019 18:08:33 +0300 Subject: [PATCH 0272/2548] release-common: fix build with `config.allowAliases = false` ```sh > nix build -f release.nix build.x86_64-linux error: undefined variable 'docbook5_xsl' at /vcs/nix/release-common.nix:45:7 ``` --- release-common.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-common.nix b/release-common.nix index 4c5565985..2e8a951b9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -42,7 +42,7 @@ rec { libxml2 libxslt docbook5 - docbook5_xsl + docbook_xsl_ns autoconf-archive autoreconfHook ]; From 094539ef4a637bde795bf67ddcc8f1f7443499f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Wed, 29 May 2019 23:09:23 +0200 Subject: [PATCH 0273/2548] Rename requires -> inputs, provides -> outputs Issue #2828. --- doc/flakes/design.md | 37 +++++++-------- flake.lock | 8 ++-- flake.nix | 10 ++-- src/libexpr/primops/flake.cc | 90 ++++++++++++++++++------------------ src/libexpr/primops/flake.hh | 6 +-- src/nix/flake.cc | 20 ++++---- src/nix/installables.cc | 14 +++--- tests/flakes.sh | 38 +++++++-------- 8 files changed, 112 insertions(+), 111 deletions(-) diff --git a/doc/flakes/design.md b/doc/flakes/design.md index ebc229b09..4f6524f83 100644 --- a/doc/flakes/design.md +++ b/doc/flakes/design.md @@ -103,12 +103,12 @@ module. # A list of flake references denoting the flakes that this flake # depends on. Nix will resolve and fetch these flakes and pass them - # as a function argument to `provides` below. + # as a function argument to `outputs` below. # # `flake:nixpkgs` denotes a flake named `nixpkgs` which is looked up # in the flake registry, or in `flake.lock` inside this flake, if it # exists. - requires = [ flake:nixpkgs ]; + inputs = [ flake:nixpkgs ]; # The stuff provided by this flake. Flakes can provide whatever they # want (convention over configuration), but some attributes have @@ -117,9 +117,9 @@ module. # `nixosModules` is used by NixOS to automatically pull in the # modules provided by a flake. # - # `provides` takes a single argument named `deps` that contains + # `outputs` takes a single argument named `deps` that contains # the resolved set of flakes. (See below.) - provides = deps: { + outputs = deps: { # This is searched by `nix`, so something like `nix install # dwarffs.dwarffs` resolves to this `packages.dwarffs`. @@ -168,7 +168,7 @@ Similarly, a minimal `flake.nix` for Nixpkgs: description = "A collection of packages for the Nix package manager"; - provides = deps: + outputs = deps: let pkgs = import ./. {}; in { lib = import ./lib; @@ -310,9 +310,9 @@ Example: ``` -## `provides` +## `outputs` -The flake attribute `provides` is a function that takes an argument +The flake attribute `outputs` is a function that takes an argument named `deps` and returns a (mostly) arbitrary attrset of values. Some of the standard result attributes: @@ -329,13 +329,13 @@ of the standard result attributes: we need to avoid a situation where `nixos-rebuild` needs to fetch its own `nixpkgs` just to do `evalModules`.) -* `shell`: A specification of a development environment in some TBD +* `devShell`: A specification of a development environment in some TBD format. The function argument `flakes` is an attrset that contains an -attribute for each dependency specified in `requires`. (Should it +attribute for each dependency specified in `inputs`. (Should it contain transitive dependencies? Probably not.) Each attribute is an -attrset containing the `provides` of the dependency, in addition to +attrset containing the `outputs` of the dependency, in addition to the following attributes: * `path`: The path to the flake's source code. Useful when you want to @@ -366,13 +366,13 @@ It may be useful to pull in repositories that are not flakes (i.e. don't contain a `flake.nix`). This could be done in two ways: * Allow flakes not to have a `flake.nix` file, in which case it's a - flake with no requires and no provides. The downside of this + flake with no inputs and no outputs. The downside of this approach is that we can't detect accidental use of a non-flake repository. (Also, we need to conjure up an identifier somehow.) * Add a flake attribute to specifiy non-flake dependencies, e.g. - > nonFlakeRequires.foobar = github:foo/bar; + > nonFlakeInputs.foobar = github:foo/bar; ## Flake registry @@ -454,7 +454,7 @@ The default installation source in `nix` is the `packages` from all flakes in the registry, that is: ``` builtins.mapAttrs (flakeName: flakeInfo: - (getFlake flakeInfo.uri).${flakeName}.provides.packages or {}) + (getFlake flakeInfo.uri).${flakeName}.outputs.packages or {}) builtins.flakeRegistry ``` (where `builtins.flakeRegistry` is the global registry with user @@ -476,10 +476,11 @@ in the registry named `hello`. Maybe the command -> nix shell +> nix dev-shell -should do something like use `provides.shell` to initialize the shell, -but probably we should ditch `nix shell` / `nix-shell` for direnv. +should do something like use `outputs.devShell` to initialize the +shell, but probably we should ditch `nix shell` / `nix-shell` for +direnv. ## Pure evaluation and caching @@ -535,7 +536,7 @@ repositories. ```nix { - provides = flakes: { + outputs = flakes: { nixosSystems.default = flakes.nixpkgs.lib.evalModules { modules = @@ -549,7 +550,7 @@ repositories. }; }; - requires = + inputs = [ "nixpkgs/nixos-18.09" "dwarffs" "hydra" diff --git a/flake.lock b/flake.lock index 965d038ce..b85571b61 100644 --- a/flake.lock +++ b/flake.lock @@ -1,10 +1,10 @@ { - "nonFlakeRequires": {}, - "requires": { + "inputs": { "nixpkgs": { - "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", - "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" + "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", + "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" } }, + "nonFlakeInputs": {}, "version": 1 } diff --git a/flake.nix b/flake.nix index 20ad2bc7e..ab96d5c90 100644 --- a/flake.nix +++ b/flake.nix @@ -5,13 +5,13 @@ epoch = 2019; - requires = [ "nixpkgs" ]; + inputs = [ "nixpkgs" ]; - provides = deps: rec { + outputs = inputs: rec { hydraJobs = import ./release.nix { - nix = deps.self; - nixpkgs = deps.nixpkgs; + nix = inputs.self; + nixpkgs = inputs.nixpkgs; }; checks = { @@ -29,7 +29,7 @@ defaultPackage = packages.nix; devShell = import ./shell.nix { - nixpkgs = deps.nixpkgs; + nixpkgs = inputs.nixpkgs; }; }; } diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 0d762f2a1..fdbdc83bc 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -56,21 +56,21 @@ LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) if (!flakeRef.isImmutable()) throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); - LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["contentHash"])); + LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["narHash"])); - auto nonFlakeRequires = json["nonFlakeRequires"]; + auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); if (!flakeRef.isImmutable()) throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } - auto requires = json["requires"]; + auto inputs = json["inputs"]; - for (auto i = requires.begin(); i != requires.end(); ++i) + for (auto i = inputs.begin(); i != inputs.end(); ++i) entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); return entry; @@ -89,19 +89,19 @@ LockFile readLockFile(const Path & path) if (version != 1) throw Error("lock file '%s' has unsupported version %d", path, version); - auto nonFlakeRequires = json["nonFlakeRequires"]; + auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeRequires.begin(); i != nonFlakeRequires.end(); ++i) { + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { FlakeRef flakeRef(i->value("uri", "")); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("contentHash", ""))); + LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); if (!flakeRef.isImmutable()) throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); } - auto requires = json["requires"]; + auto inputs = json["inputs"]; - for (auto i = requires.begin(); i != requires.end(); ++i) + for (auto i = inputs.begin(); i != inputs.end(); ++i) lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); return lockFile; @@ -111,13 +111,13 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) { nlohmann::json json; json["uri"] = entry.ref.to_string(); - json["contentHash"] = entry.narHash.to_string(SRI); + json["narHash"] = entry.narHash.to_string(SRI); for (auto & x : entry.nonFlakeEntries) { - json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); + json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); } for (auto & x : entry.flakeEntries) - json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); + json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); return json; } @@ -125,14 +125,14 @@ void writeLockFile(const LockFile & lockFile, const Path & path) { nlohmann::json json; json["version"] = 1; - json["nonFlakeRequires"] = nlohmann::json::object(); + json["nonFlakeInputs"] = nlohmann::json::object(); for (auto & x : lockFile.nonFlakeEntries) { - json["nonFlakeRequires"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeRequires"][x.first]["contentHash"] = x.second.narHash.to_string(SRI); + json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); + json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); } - json["requires"] = nlohmann::json::object(); + json["inputs"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) - json["requires"][x.first.to_string()] = flakeEntryToJson(x.second); + json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); createDirs(dirOf(path)); writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file } @@ -319,41 +319,41 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe if (auto description = vInfo.attrs->get(state.sDescription)) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); - auto sRequires = state.symbols.create("requires"); + auto sInputs = state.symbols.create("inputs"); - if (auto requires = vInfo.attrs->get(sRequires)) { - state.forceList(*(**requires).value, *(**requires).pos); - for (unsigned int n = 0; n < (**requires).value->listSize(); ++n) - flake.requires.push_back(FlakeRef(state.forceStringNoCtx( - *(**requires).value->listElems()[n], *(**requires).pos))); + if (auto inputs = vInfo.attrs->get(sInputs)) { + state.forceList(*(**inputs).value, *(**inputs).pos); + for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) + flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( + *(**inputs).value->listElems()[n], *(**inputs).pos))); } - auto sNonFlakeRequires = state.symbols.create("nonFlakeRequires"); + auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); - if (std::optional<Attr *> nonFlakeRequires = vInfo.attrs->get(sNonFlakeRequires)) { - state.forceAttrs(*(**nonFlakeRequires).value, *(**nonFlakeRequires).pos); - for (Attr attr : *(*(**nonFlakeRequires).value).attrs) { + if (std::optional<Attr *> nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { + state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); + for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); - flake.nonFlakeRequires.insert_or_assign(attr.name, nonFlakeRef); + flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); } } - auto sProvides = state.symbols.create("provides"); + auto sOutputs = state.symbols.create("outputs"); - if (auto provides = vInfo.attrs->get(sProvides)) { - state.forceFunction(*(**provides).value, *(**provides).pos); - flake.vProvides = (**provides).value; + if (auto outputs = vInfo.attrs->get(sOutputs)) { + state.forceFunction(*(**outputs).value, *(**outputs).pos); + flake.vOutputs = (**outputs).value; } else - throw Error("flake '%s' lacks attribute 'provides'", flakeRef); + throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); for (auto & attr : *vInfo.attrs) { if (attr.name != sEpoch && attr.name != state.sName && attr.name != state.sDescription && - attr.name != sRequires && - attr.name != sNonFlakeRequires && - attr.name != sProvides) + attr.name != sInputs && + attr.name != sNonFlakeInputs && + attr.name != sOutputs) throw Error("flake '%s' has an unsupported attribute '%s', at %s", flakeRef, attr.name, *attr.pos); } @@ -436,7 +436,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake ResolvedFlake deps(flake); - for (auto & nonFlakeInfo : flake.nonFlakeRequires) { + for (auto & nonFlakeInfo : flake.nonFlakeInputs) { FlakeRef ref = nonFlakeInfo.second; auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); if (i != lockFile.nonFlakeEntries.end()) { @@ -451,7 +451,7 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake } } - for (auto newFlakeRef : flake.requires) { + for (auto newFlakeRef : flake.inputs) { auto i = lockFile.flakeEntries.find(newFlakeRef); if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); @@ -532,8 +532,8 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) { - // Construct the resulting attrset '{description, provides, - // ...}'. This attrset is passed lazily as an argument to 'provides'. + // Construct the resulting attrset '{description, outputs, + // ...}'. This attrset is passed lazily as an argument to 'outputs'. state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); @@ -558,8 +558,8 @@ void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v); - auto vProvides = state.allocAttr(v, state.symbols.create("provides")); - mkApp(*vProvides, *resFlake.flake.vProvides, v); + auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); + mkApp(*vOutputs, *resFlake.flake.vOutputs, v); v.attrs->push_back(Attr(state.symbols.create("self"), &v)); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 340b97c65..82b0973f6 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -106,9 +106,9 @@ struct Flake FlakeRef originalRef; std::string description; SourceInfo sourceInfo; - std::vector<FlakeRef> requires; - std::map<FlakeAlias, FlakeRef> nonFlakeRequires; - Value * vProvides; // FIXME: gc + std::vector<FlakeRef> inputs; + std::map<FlakeAlias, FlakeRef> nonFlakeInputs; + Value * vOutputs; // FIXME: gc unsigned int epoch; Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 872ec2849..8d6716391 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -199,16 +199,16 @@ struct CmdFlakeUpdate : FlakeCommand } }; -static void enumerateProvides(EvalState & state, Value & vFlake, +static void enumerateOutputs(EvalState & state, Value & vFlake, std::function<void(const std::string & name, Value & vProvide)> callback) { state.forceAttrs(vFlake); - auto vProvides = (*vFlake.attrs->get(state.symbols.create("provides")))->value; + auto vOutputs = (*vFlake.attrs->get(state.symbols.create("outputs")))->value; - state.forceAttrs(*vProvides); + state.forceAttrs(*vOutputs); - for (auto & attr : *vProvides->attrs) + for (auto & attr : *vOutputs->attrs) callback(attr.name, *attr.value); } @@ -237,9 +237,9 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON auto vFlake = state->allocValue(); flake::callFlake(*state, flake, *vFlake); - auto provides = nlohmann::json::object(); + auto outputs = nlohmann::json::object(); - enumerateProvides(*state, + enumerateOutputs(*state, *vFlake, [&](const std::string & name, Value & vProvide) { auto provide = nlohmann::json::object(); @@ -250,10 +250,10 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON provide[aCheck.name] = nlohmann::json::object(); } - provides[name] = provide; + outputs[name] = provide; }); - json["provides"] = std::move(provides); + json["outputs"] = std::move(outputs); std::cout << json.dump() << std::endl; } else @@ -298,7 +298,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON // FIXME: check meta attributes return drvInfo->queryDrvPath(); } catch (Error & e) { - e.addPrefix(fmt("while checking flake attribute '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + e.addPrefix(fmt("while checking flake output attribute '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); throw; } }; @@ -311,7 +311,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto vFlake = state->allocValue(); flake::callFlake(*state, flake, *vFlake); - enumerateProvides(*state, + enumerateOutputs(*state, *vFlake, [&](const std::string & name, Value & vProvide) { Activity act(*logger, lvlChatty, actUnknown, diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ea12cd79c..fe89a6bb4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -230,16 +230,16 @@ struct InstallableFlake : InstallableValue makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); - auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; + auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; - state.forceValue(*vProvides); + state.forceValue(*vOutputs); auto emptyArgs = state.allocBindings(0); if (searchPackages) { // As a convenience, look for the attribute in - // 'provides.packages'. - if (auto aPackages = *vProvides->attrs->get(state.symbols.create("packages"))) { + // 'outputs.packages'. + if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) { try { auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); state.forceValue(*v); @@ -250,7 +250,7 @@ struct InstallableFlake : InstallableValue // As a temporary hack until Nixpkgs is properly converted // to provide a clean 'packages' set, look in 'legacyPackages'. - if (auto aPackages = *vProvides->attrs->get(state.symbols.create("legacyPackages"))) { + if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) { try { auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); state.forceValue(*v); @@ -260,10 +260,10 @@ struct InstallableFlake : InstallableValue } } - // Otherwise, look for it in 'provides'. + // Otherwise, look for it in 'outputs'. for (auto & attrPath : attrPaths) { try { - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vProvides); + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); state.forceValue(*v); return v; } catch (AttrPathNotFound & e) { diff --git a/tests/flakes.sh b/tests/flakes.sh index d95d34c76..377f93c8e 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -33,7 +33,7 @@ cat > $flake1Dir/flake.nix <<EOF description = "Bla bla"; - provides = deps: rec { + outputs = inputs: rec { packages.foo = import ./simple.nix; defaultPackage = packages.foo; }; @@ -50,12 +50,12 @@ cat > $flake2Dir/flake.nix <<EOF epoch = 2019; - requires = [ "flake1" ]; + inputs = [ "flake1" ]; description = "Fnord"; - provides = deps: rec { - packages.bar = deps.flake1.provides.packages.foo; + outputs = inputs: rec { + packages.bar = inputs.flake1.outputs.packages.foo; }; } EOF @@ -69,12 +69,12 @@ cat > $flake3Dir/flake.nix <<EOF epoch = 2019; - requires = [ "flake2" ]; + inputs = [ "flake2" ]; description = "Fnord"; - provides = deps: rec { - packages.xyzzy = deps.flake2.provides.packages.bar; + outputs = inputs: rec { + packages.xyzzy = inputs.flake2.outputs.packages.bar; }; } EOF @@ -168,13 +168,13 @@ cat > $flake3Dir/flake.nix <<EOF epoch = 2019; - requires = [ "flake1" "flake2" ]; + inputs = [ "flake1" "flake2" ]; description = "Fnord"; - provides = deps: rec { - packages.xyzzy = deps.flake2.provides.packages.bar; - packages.sth = deps.flake1.provides.packages.foo; + outputs = inputs: rec { + packages.xyzzy = inputs.flake2.outputs.packages.bar; + packages.sth = inputs.flake1.outputs.packages.foo; }; } EOF @@ -209,7 +209,7 @@ nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2D mv $flake1Dir.tmp $flake1Dir mv $flake2Dir.tmp $flake2Dir -# Add nonFlakeRequires to flake3. +# Add nonFlakeInputs to flake3. rm $flake3Dir/flake.nix cat > $flake3Dir/flake.nix <<EOF @@ -218,23 +218,23 @@ cat > $flake3Dir/flake.nix <<EOF epoch = 2019; - requires = [ "flake1" "flake2" ]; + inputs = [ "flake1" "flake2" ]; - nonFlakeRequires = { + nonFlakeInputs = { nonFlake = "$nonFlakeDir"; }; description = "Fnord"; - provides = deps: rec { - packages.xyzzy = deps.flake2.provides.packages.bar; - packages.sth = deps.flake1.provides.packages.foo; + outputs = inputs: rec { + packages.xyzzy = inputs.flake2.outputs.packages.bar; + packages.sth = inputs.flake1.outputs.packages.foo; }; } EOF git -C $flake3Dir add flake.nix -git -C $flake3Dir commit -m 'Add nonFlakeRequires' +git -C $flake3Dir commit -m 'Add nonFlakeInputs' -# Check whether `nix build` works with a lockfile which is missing a nonFlakeRequires +# Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth From 63c5c91cc053cbc1fcb8d3fe71c41142c9f51bfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 18:48:28 +0200 Subject: [PATCH 0274/2548] Show hash mismatch warnings in SRI format --- src/libstore/build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 004be8010..a69592219 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3170,7 +3170,7 @@ void DerivationGoal::registerOutputs() valid. */ delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(), h2.to_string())); + dest, h.to_string(SRI), h2.to_string(SRI))); Path actualDest = worker.store.toRealPath(dest); From b971e406dee82486d53737a928b0bb3b482a6c49 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 19:01:11 +0200 Subject: [PATCH 0275/2548] Support 'dir' and other parameters in path flakerefs --- src/libexpr/primops/flakeref.cc | 90 ++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 41 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 3c805eff8..306da20fe 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -1,4 +1,5 @@ #include "flakeref.hh" +#include "store-api.hh" #include <regex> @@ -30,10 +31,6 @@ const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; -// FIXME: support escaping in query string. -// Note: '/' is not a valid query parameter, but so what... -const static std::string paramRegex = "[a-z]+=[/a-zA-Z0-9._-]*"; -const static std::string paramsRegex = "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))"; // 'dir' path elements cannot start with a '.'. We also reject // potentially dangerous characters like ';'. @@ -41,7 +38,7 @@ const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; -FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) +FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) { // FIXME: could combine this into one regex. @@ -50,21 +47,46 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) std::regex::ECMAScript); static std::regex githubRegex( - "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?" - + paramsRegex + "?", + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", std::regex::ECMAScript); static std::regex uriRegex( "((" + schemeRegex + "):" + "(?://(" + authorityRegex + "))?" + - "(" + pathRegex + "))" + - paramsRegex + "?", + "(" + pathRegex + "))", std::regex::ECMAScript); static std::regex refRegex2(refRegex, std::regex::ECMAScript); static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); + auto [uri, params] = splitUriAndParams(uri_); + + auto handleSubdir = [&](const std::string & name, const std::string & value) { + if (name == "dir") { + if (value != "" && !std::regex_match(value, subDirRegex2)) + throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); + subdir = value; + return true; + } else + return false; + }; + + auto handleGitParams = [&](const std::string & name, const std::string & value) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw Error("invalid Git revision '%s'", value); + rev = Hash(value, htSHA1); + } else if (name == "ref") { + if (!std::regex_match(value, refRegex2)) + throw Error("invalid Git ref '%s'", value); + ref = value; + } else if (handleSubdir(name, value)) + ; + else return false; + return true; + }; + std::cmatch match; if (std::regex_match(uri.c_str(), match, flakeRegex)) { IsAlias d; @@ -88,17 +110,11 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) else if (match[4].matched) { ref = match[4]; } - for (auto & param : tokenizeString<Strings>(match[5], "&")) { - auto n = param.find('='); - assert(n != param.npos); - std::string name(param, 0, n); - std::string value(param, n + 1); - if (name == "dir") { - if (value != "" && !std::regex_match(value, subDirRegex2)) - throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); - subdir = value; - } else - throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); + for (auto & param : params) { + if (handleSubdir(param.first, param.second)) + ; + else + throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); } data = d; } @@ -108,26 +124,12 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) { IsGit d; d.uri = match[1]; - for (auto & param : tokenizeString<Strings>(match[5], "&")) { - auto n = param.find('='); - assert(n != param.npos); - std::string name(param, 0, n); - std::string value(param, n + 1); - if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw Error("invalid Git revision '%s'", value); - rev = Hash(value, htSHA1); - } else if (name == "ref") { - if (!std::regex_match(value, refRegex2)) - throw Error("invalid Git ref '%s'", value); - ref = value; - } else if (name == "dir") { - if (value != "" && !std::regex_match(value, subDirRegex2)) - throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); - subdir = value; - } else + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else // FIXME: should probably pass through unknown parameters - throw Error("invalid Git flake reference parameter '%s', in '%s'", name, uri); + throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); } if (rev && !ref) throw Error("flake URI '%s' lacks a Git ref", uri); @@ -138,6 +140,12 @@ FlakeRef::FlakeRef(const std::string & uri, bool allowRelative) IsPath d; d.path = allowRelative ? absPath(uri) : canonPath(uri); data = d; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } } else @@ -165,10 +173,10 @@ std::string FlakeRef::to_string() const } else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) { - assert(subdir == ""); + string = refData->path; if (ref) addParam("ref", *ref); if (rev) addParam("rev", rev->gitRev()); - return refData->path; + if (subdir != "") addParam("dir", subdir); } else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) { From 9169046e64cbffd85a445ebe4313b172a6681646 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 20:10:56 +0200 Subject: [PATCH 0276/2548] Add operator << for LockFile Useful for debugging. --- src/libexpr/primops/flake.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index fdbdc83bc..af7d51834 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -121,7 +121,7 @@ nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) return json; } -void writeLockFile(const LockFile & lockFile, const Path & path) +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { nlohmann::json json; json["version"] = 1; @@ -133,8 +133,14 @@ void writeLockFile(const LockFile & lockFile, const Path & path) json["inputs"] = nlohmann::json::object(); for (auto & x : lockFile.flakeEntries) json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); + stream << json.dump(4); // '4' = indentation in json file + return stream; +} + +void writeLockFile(const LockFile & lockFile, const Path & path) +{ createDirs(dirOf(path)); - writeFile(path, json.dump(4) + "\n"); // '4' = indentation in json file + writeFile(path, fmt("%s\n", lockFile)); } Path getUserRegistryPath() From 7adb10d29b0041a93d1afeec197bf9af6e8b25b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 20:12:59 +0200 Subject: [PATCH 0277/2548] Fix reading the lockfile of a flake in a subdirectory --- src/libexpr/primops/flake.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index af7d51834..235e10922 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -482,9 +482,12 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); LockFile oldLockFile; - if (!recreateLockFile (handleLockFile)) { + if (!recreateLockFile(handleLockFile)) { // If recreateLockFile, start with an empty lockfile - oldLockFile = readLockFile(flake.sourceInfo.storePath + "/flake.lock"); // FIXME: symlink attack + // FIXME: symlink attack + oldLockFile = readLockFile( + state.store->toRealPath(flake.sourceInfo.storePath) + + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } LockFile lockFile(oldLockFile); From ccb1bad612e060fc4397d340edc64d18231744b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 20:53:23 +0200 Subject: [PATCH 0278/2548] Allow bare flakerefs as installables So now $ nix build blender-bin works and builds the default package from that flake. You don't need to add a colon at the end anymore. --- src/libexpr/primops/flakeref.cc | 31 ++++++++++++++++++++++--------- src/libexpr/primops/flakeref.hh | 5 +++++ src/nix/installables.cc | 4 ++++ tests/flakes.sh | 2 +- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 306da20fe..4b6922295 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -65,7 +65,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) auto handleSubdir = [&](const std::string & name, const std::string & value) { if (name == "dir") { if (value != "" && !std::regex_match(value, subDirRegex2)) - throw Error("flake '%s' has invalid subdirectory '%s'", uri, value); + throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); subdir = value; return true; } else @@ -75,11 +75,11 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) auto handleGitParams = [&](const std::string & name, const std::string & value) { if (name == "rev") { if (!std::regex_match(value, revRegex)) - throw Error("invalid Git revision '%s'", value); + throw BadFlakeRef("invalid Git revision '%s'", value); rev = Hash(value, htSHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex2)) - throw Error("invalid Git ref '%s'", value); + throw BadFlakeRef("invalid Git ref '%s'", value); ref = value; } else if (handleSubdir(name, value)) ; @@ -114,7 +114,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) if (handleSubdir(param.first, param.second)) ; else - throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); } data = d; } @@ -129,14 +129,16 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) ; else // FIXME: should probably pass through unknown parameters - throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); } if (rev && !ref) - throw Error("flake URI '%s' lacks a Git ref", uri); + throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); data = d; } - else if (hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) { + else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) + && uri.find(':') == std::string::npos) + { IsPath d; d.path = allowRelative ? absPath(uri) : canonPath(uri); data = d; @@ -144,12 +146,12 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) if (handleGitParams(param.first, param.second)) ; else - throw Error("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); } } else - throw Error("'%s' is not a valid flake reference", uri); + throw BadFlakeRef("'%s' is not a valid flake reference", uri); } std::string FlakeRef::to_string() const @@ -225,4 +227,15 @@ FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. result.rev = std::nullopt; return result; } + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative) +{ + try { + return FlakeRef(uri, allowRelative); + } catch (BadFlakeRef & e) { + return {}; + } +} + } diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh index 299094634..52bb82ddb 100644 --- a/src/libexpr/primops/flakeref.hh +++ b/src/libexpr/primops/flakeref.hh @@ -180,4 +180,9 @@ struct FlakeRef std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); +MakeError(BadFlakeRef, Error); + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative = false); + } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index fe89a6bb4..40248eb5d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -332,6 +332,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( getDefaultFlakeAttrPaths())); } + else if (auto flakeRef = parseFlakeRef(s, true)) + result.push_back(std::make_shared<InstallableFlake>(*this, s, + getDefaultFlakeAttrPaths())); + else result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef("nixpkgs"), s)); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 377f93c8e..8b9cb7260 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -131,7 +131,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo [[ -e $TEST_ROOT/result/hello ]] # Test defaultPackage. -nix build -o $TEST_ROOT/result --flake-registry $registry flake1: +nix build -o $TEST_ROOT/result --flake-registry $registry flake1 [[ -e $TEST_ROOT/result/hello ]] # Building a flake with an unlocked dependency should fail in pure mode. From 8abb8647a33c3516026cd8a2954d34633377b23c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 21:52:02 +0200 Subject: [PATCH 0279/2548] Automatically determine subdir for path flakes This means that in a flake in a subdirectory of a Git repo, you can now do $ nix build rather than the inconvenient $ nix build ../..?dir=foo/bar --- src/libexpr/primops/flakeref.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 4b6922295..6c90c3b64 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -140,7 +140,17 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) && uri.find(':') == std::string::npos) { IsPath d; - d.path = allowRelative ? absPath(uri) : canonPath(uri); + if (allowRelative) { + d.path = absPath(uri); + while (true) { + if (pathExists(d.path + "/.git")) break; + subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); + d.path = dirOf(d.path); + if (d.path == "/") + throw BadFlakeRef("path '%s' does not reference a Git repository", uri); + } + } else + d.path = canonPath(uri); data = d; for (auto & param : params) { if (handleGitParams(param.first, param.second)) From 8cb3bbd5044b8fbfc65f13455d1619a78ccf33a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 22:17:39 +0200 Subject: [PATCH 0280/2548] Fix handling of bare flakerefs containing a colon --- src/nix/installables.cc | 8 ++++---- tests/flakes.sh | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 40248eb5d..38ae416e3 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -314,6 +314,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( Strings{"packages." + std::string(s, 8)})); } + else if (auto flakeRef = parseFlakeRef(s, true)) + result.push_back(std::make_shared<InstallableFlake>(*this, s, + getDefaultFlakeAttrPaths())); + else if ((colon = s.rfind(':')) != std::string::npos) { auto flakeRef = std::string(s, 0, colon); auto attrPath = std::string(s, colon + 1); @@ -332,10 +336,6 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( getDefaultFlakeAttrPaths())); } - else if (auto flakeRef = parseFlakeRef(s, true)) - result.push_back(std::make_shared<InstallableFlake>(*this, s, - getDefaultFlakeAttrPaths())); - else result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef("nixpkgs"), s)); } diff --git a/tests/flakes.sh b/tests/flakes.sh index 8b9cb7260..c4dd8c333 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -134,6 +134,9 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo nix build -o $TEST_ROOT/result --flake-registry $registry flake1 [[ -e $TEST_ROOT/result/hello ]] +nix build -o $TEST_ROOT/result --flake-registry $registry $flake1Dir +nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake1Dir + # Building a flake with an unlocked dependency should fail in pure mode. (! nix eval "(builtins.getFlake "$flake2Dir")") From 15f241775ace2bbd807e7222e822bd5bf0f42ff7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 23:21:53 +0200 Subject: [PATCH 0281/2548] Doh --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 38ae416e3..eb3c27d6b 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -315,7 +315,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } else if (auto flakeRef = parseFlakeRef(s, true)) - result.push_back(std::make_shared<InstallableFlake>(*this, s, + result.push_back(std::make_shared<InstallableFlake>(*this, std::move(*flakeRef), getDefaultFlakeAttrPaths())); else if ((colon = s.rfind(':')) != std::string::npos) { From 2d34028b1e4e46d600926d4ad0c687c4d1d58c75 Mon Sep 17 00:00:00 2001 From: worldofpeace <worldofpeace@protonmail.ch> Date: Fri, 31 May 2019 22:19:46 -0400 Subject: [PATCH 0282/2548] Add .github/FUNDING.yml --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..bbaabf93c --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +custom: https://nixos.org/nixos/foundation.html From aec545c20b15eace71c6733107dd6fde7736afbe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Sat, 1 Jun 2019 15:27:43 +0200 Subject: [PATCH 0283/2548] Fix segfault in builtin fetchurl with hashed mirrors + SRI hashes --- src/libstore/builtins/fetchurl.cc | 3 ++- tests/fetchurl.sh | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 92aec63a0..b1af3b4fc 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index ec3399b08..7319ced2b 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,6 +42,10 @@ ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +# Test hashed mirrors with an SRI hash. +nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + # Test unpacking a NAR. rm -rf $TEST_ROOT/archive mkdir -p $TEST_ROOT/archive From fb692e5f7b34def8cf590298036ab63e40747062 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 23:44:42 +0200 Subject: [PATCH 0284/2548] Bindings: Add convenience method for requiring an attribute --- src/libexpr/attr-set.hh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 6c5fb21ad..d6af99912 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -72,6 +72,14 @@ public: return {}; } + Attr & need(const Symbol & name, const Pos & pos = noPos) + { + auto a = get(name); + if (!a) + throw Error("attribute '%s' missing, at %s", name, pos); + return **a; + } + iterator begin() { return &attrs[0]; } iterator end() { return &attrs[size_]; } From 5fbd9fee0b4b26cc7bcceb350e56e808c7a70e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Fri, 31 May 2019 23:45:13 +0200 Subject: [PATCH 0285/2548] Add 'nix app' command This is like 'nix run', except that the command to execute is defined in a flake output, e.g. defaultApp = { type = "app"; program = "${packages.blender_2_80}/bin/blender"; }; Thus you can do $ nix app blender-bin to start Blender from the 'blender-bin' flake. In the future, we can extend this with sandboxing. (For example we would want to be able to specify that Blender should not have network access by default and should only have access to certain paths in the user's home directory.) --- src/nix/command.hh | 9 ++++ src/nix/installables.cc | 22 ++++++++ src/nix/run.cc | 115 +++++++++++++++++++++++++++++----------- 3 files changed, 116 insertions(+), 30 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index 26c308331..659b724c3 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -38,6 +38,13 @@ struct Buildable typedef std::vector<Buildable> Buildables; +struct App +{ + PathSet context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + struct Installable { virtual std::string what() = 0; @@ -49,6 +56,8 @@ struct Installable Buildable toBuildable(); + App toApp(EvalState & state); + virtual Value * toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index eb3c27d6b..b6f05b314 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -68,6 +68,28 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } +App Installable::toApp(EvalState & state) +{ + auto v = toValue(state); + + state.forceAttrs(*v); + + auto aType = v->attrs->need(state.sType); + if (state.forceStringNoCtx(*aType.value, *aType.pos) != "app") + throw Error("value does not have type 'app', at %s", *aType.pos); + + App app; + + auto aProgram = v->attrs->need(state.symbols.create("program")); + app.program = state.forceString(*aProgram.value, app.context, *aProgram.pos); + + // FIXME: check that 'program' is in the closure of 'context'. + if (!state.store->isInStore(app.program)) + throw Error("app program '%s' is not in the Nix store", app.program); + + return app; +} + struct InstallableStorePath : Installable { Path storePath; diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345..00a682832 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -8,6 +8,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "affinity.hh" +#include "eval.hh" #if __linux__ #include <sys/mount.h> @@ -19,7 +20,44 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct CmdRun : InstallablesCommand +struct RunCommon : virtual Command +{ + void runProgram(ref<Store> store, + const std::string & program, + const Strings & args) + { + stopProgressBar(); + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); + } +}; + +struct CmdRun : InstallablesCommand, RunCommon { std::vector<std::string> command = { "bash" }; StringSet keep, unset; @@ -147,43 +185,60 @@ struct CmdRun : InstallablesCommand setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - std::string cmd = *command.begin(); Strings args; for (auto & arg : command) args.push_back(arg); - stopProgressBar(); - - restoreSignals(); - - restoreAffinity(); - - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); - - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; - for (auto & arg : args) helperArgs.push_back(arg); - - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - - throw SysError("could not execute chroot helper"); - } - - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("unable to exec '%s'", cmd); + runProgram(store, *command.begin(), args); } }; static RegisterCommand r1(make_ref<CmdRun>()); +struct CmdApp : InstallableCommand, RunCommon +{ + CmdApp() + { + } + + std::string name() override + { + return "app"; + } + + std::string description() override + { + return "run a Nix application"; + } + + Examples examples() override + { + return { + Example{ + "To run Blender:", + "nix app blender-bin" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + return {"defaultApp"}; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + auto app = installable->toApp(*state); + + state->realiseContext(app.context); + + runProgram(store, app.program, {app.program}); + } +}; + +static RegisterCommand r2(make_ref<CmdApp>()); + void chrootHelper(int argc, char * * argv) { int p = 1; From d9a6a75ed28c590dde2dba846e356cbcda38d977 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck <nick.van.den.broeck666@gmail.com> Date: Mon, 3 Jun 2019 14:47:47 +0200 Subject: [PATCH 0286/2548] Made epochs more fine-grained Fixes #2894 --- doc/flakes/design.md | 4 ++-- flake.nix | 2 +- src/libexpr/primops/flake.cc | 2 +- src/nix/flake-template.nix | 2 +- tests/flakes.sh | 12 ++++++------ 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/flakes/design.md b/doc/flakes/design.md index 4f6524f83..63198e577 100644 --- a/doc/flakes/design.md +++ b/doc/flakes/design.md @@ -96,7 +96,7 @@ module. # The epoch may be used in the future to determine how Nix # expressions inside this flake are to be parsed. - epoch = 2018; + epoch = 201906; # Some other metadata. description = "A filesystem that fetches DWARF debug info from the Internet on demand"; @@ -164,7 +164,7 @@ Similarly, a minimal `flake.nix` for Nixpkgs: { name = "nixpkgs"; - epoch = 2018; + epoch = 201906; description = "A collection of packages for the Nix package manager"; diff --git a/flake.nix b/flake.nix index ab96d5c90..e7deb2de3 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ description = "The purely functional package manager"; - epoch = 2019; + epoch = 201906; inputs = [ "nixpkgs" ]; diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 235e10922..e5035c53a 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -312,7 +312,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe if (auto epoch = vInfo.attrs->get(sEpoch)) { flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); - if (flake.epoch > 2019) + if (flake.epoch > 201906) throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); } else throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); diff --git a/src/nix/flake-template.nix b/src/nix/flake-template.nix index fe89e647e..bec613f6c 100644 --- a/src/nix/flake-template.nix +++ b/src/nix/flake-template.nix @@ -3,7 +3,7 @@ description = "A flake for building Hello World"; - epoch = 2019; + epoch = 201906; requires = [ "nixpkgs" ]; diff --git a/tests/flakes.sh b/tests/flakes.sh index c4dd8c333..998abfd09 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -29,7 +29,7 @@ cat > $flake1Dir/flake.nix <<EOF { name = "flake1"; - epoch = 2019; + epoch = 201906; description = "Bla bla"; @@ -48,7 +48,7 @@ cat > $flake2Dir/flake.nix <<EOF { name = "flake2"; - epoch = 2019; + epoch = 201906; inputs = [ "flake1" ]; @@ -67,7 +67,7 @@ cat > $flake3Dir/flake.nix <<EOF { name = "flake3"; - epoch = 2019; + epoch = 201906; inputs = [ "flake2" ]; @@ -169,7 +169,7 @@ cat > $flake3Dir/flake.nix <<EOF { name = "flake3"; - epoch = 2019; + epoch = 201906; inputs = [ "flake1" "flake2" ]; @@ -192,7 +192,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth [[ ! (-z $(git -C $flake3Dir diff master)) ]] # Unsupported epochs should be an error. -sed -i $flake3Dir/flake.nix -e s/2019/2030/ +sed -i $flake3Dir/flake.nix -e s/201906/201909/ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch' # Test whether registry caching works. @@ -219,7 +219,7 @@ cat > $flake3Dir/flake.nix <<EOF { name = "flake3"; - epoch = 2019; + epoch = 201906; inputs = [ "flake1" "flake2" ]; From 653c4e439bbd7649f7e508ed56e54e8d78b5e777 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Mon, 3 Jun 2019 21:51:06 +0200 Subject: [PATCH 0287/2548] Fix clang compilation error https://hydra.nixos.org/build/94332344 https://stackoverflow.com/questions/46114214/lambda-implicit-capture-fails-with-variable-declared-from-structured-binding --- src/libexpr/primops/flakeref.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc index 6c90c3b64..24af09124 100644 --- a/src/libexpr/primops/flakeref.cc +++ b/src/libexpr/primops/flakeref.cc @@ -60,7 +60,8 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); - auto [uri, params] = splitUriAndParams(uri_); + auto [uri2, params] = splitUriAndParams(uri_); + std::string uri(uri2); auto handleSubdir = [&](const std::string & name, const std::string & value) { if (name == "dir") { From c7c562416c75ed60e024f84e4ac440e29b98e0e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra <edolstra@gmail.com> Date: Tue, 4 Jun 2019 13:44:12 +0200 Subject: [PATCH 0288/2548] shell.nix: Use clang by default Clang compiles faster (121s vs 156s for GCC 7) so it's a bit nicer for development. --- shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index d7e63bad3..dd2950b29 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,4 @@ -{ useClang ? false +{ useClang ? true , nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz }: From 4b0d6133836113572418f383ddd14c92bfebc768 Mon Sep 17 00:00:00 2001 From: JorisE <JorisE@users.noreply.github.com> Date: Tue, 4 Jun 2019 14:12:03 +0200 Subject: [PATCH 0289/2548] Minor typo --- doc/manual/expressions/language-constructs.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index 923b5d8c4..0d0cbbe15 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para> <simplesect xml:id="sect-let-expressions"><title>Let-expressions -A let-expression allows you define local variables for an +A let-expression allows you to define local variables for an expression. For instance, From 6dbd5c26e6c853f302cd9d3ed171d134ff24ffe1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 19:10:35 +0200 Subject: [PATCH 0290/2548] Make flake input fetching lazy As long as the flake input is locked, it is now only fetched when it is evaluated (e.g. "nixpkgs" is fetched when "inputs.nixpkgs." is evaluated). This required adding an "id" attribute to the members of "inputs" in lockfiles, e.g. "inputs": { "nixpkgs/release-19.03": { "id": "nixpkgs", "inputs": {}, "narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=", "nonFlakeInputs": {}, "uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1" } } because the flake ID needs to be known beforehand to construct the "inputs" attrset. Fixes #2913. --- src/libexpr/primops/flake.cc | 292 +++++++++++++++++++---------------- src/libexpr/primops/flake.hh | 138 +++++++++++------ src/nix/flake.cc | 6 +- src/nix/installables.cc | 2 + src/nlohmann/json_fwd.hpp | 10 ++ tests/flakes.sh | 11 ++ 6 files changed, 272 insertions(+), 187 deletions(-) create mode 100644 src/nlohmann/json_fwd.hpp diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index e5035c53a..189663e51 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -43,97 +43,101 @@ std::shared_ptr readRegistry(const Path & path) void writeRegistry(const FlakeRegistry & registry, const Path & path) { nlohmann::json json; - json["version"] = 1; + json["version"] = 2; for (auto elem : registry.entries) json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -LockFile::FlakeEntry readFlakeEntry(nlohmann::json json) +NonFlakeDep::NonFlakeDep(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) { - FlakeRef flakeRef(json["uri"]); - if (!flakeRef.isImmutable()) - throw Error("cannot use mutable flake '%s' in pure mode", flakeRef); + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} - LockFile::FlakeEntry entry(flakeRef, Hash((std::string) json["narHash"])); +nlohmann::json NonFlakeDep::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} +FlakeDep::FlakeDep(const nlohmann::json & json) + : FlakeInputs(json) + , id(json["id"]) + , ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json FlakeDep::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["id"] = id; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ auto nonFlakeInputs = json["nonFlakeInputs"]; - - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - if (!flakeRef.isImmutable()) - throw Error("requested to fetch FlakeRef '%s' purely, which is mutable", flakeRef); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); - entry.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); - } + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) + nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - entry.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); + flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); +} - return entry; +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeDeps) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeDeps) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; } LockFile readLockFile(const Path & path) { - LockFile lockFile; + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); - if (!pathExists(path)) - return lockFile; + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("lock file '%s' has unsupported version %d", path, version); - - auto nonFlakeInputs = json["nonFlakeInputs"]; - - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) { - FlakeRef flakeRef(i->value("uri", "")); - LockFile::NonFlakeEntry nonEntry(flakeRef, Hash(i->value("narHash", ""))); - if (!flakeRef.isImmutable()) - throw Error("found mutable FlakeRef '%s' in lockfile at path %s", flakeRef, path); - lockFile.nonFlakeEntries.insert_or_assign(i.key(), nonEntry); - } - - auto inputs = json["inputs"]; - - for (auto i = inputs.begin(); i != inputs.end(); ++i) - lockFile.flakeEntries.insert_or_assign(i.key(), readFlakeEntry(*i)); - - return lockFile; -} - -nlohmann::json flakeEntryToJson(const LockFile::FlakeEntry & entry) -{ - nlohmann::json json; - json["uri"] = entry.ref.to_string(); - json["narHash"] = entry.narHash.to_string(SRI); - for (auto & x : entry.nonFlakeEntries) { - json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); - } - for (auto & x : entry.flakeEntries) - json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); - return json; + return LockFile(json); + } else + return LockFile(); } std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - nlohmann::json json; - json["version"] = 1; - json["nonFlakeInputs"] = nlohmann::json::object(); - for (auto & x : lockFile.nonFlakeEntries) { - json["nonFlakeInputs"][x.first]["uri"] = x.second.ref.to_string(); - json["nonFlakeInputs"][x.first]["narHash"] = x.second.narHash.to_string(SRI); - } - json["inputs"] = nlohmann::json::object(); - for (auto & x : lockFile.flakeEntries) - json["inputs"][x.first.to_string()] = flakeEntryToJson(x.second); - stream << json.dump(4); // '4' = indentation in json file + stream << lockFile.toJson().dump(4); // '4' = indentation in json file return stream; } @@ -387,33 +391,6 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al return nonFlake; } -LockFile entryToLockFile(const LockFile::FlakeEntry & entry) -{ - LockFile lockFile; - lockFile.flakeEntries = entry.flakeEntries; - lockFile.nonFlakeEntries = entry.nonFlakeEntries; - return lockFile; -} - -LockFile::FlakeEntry dependenciesToFlakeEntry(const ResolvedFlake & resolvedFlake) -{ - LockFile::FlakeEntry entry( - resolvedFlake.flake.sourceInfo.resolvedRef, - resolvedFlake.flake.sourceInfo.narHash); - - for (auto & info : resolvedFlake.flakeDeps) - entry.flakeEntries.insert_or_assign(info.first.to_string(), dependenciesToFlakeEntry(info.second)); - - for (auto & nonFlake : resolvedFlake.nonFlakeDeps) { - LockFile::NonFlakeEntry nonEntry( - nonFlake.sourceInfo.resolvedRef, - nonFlake.sourceInfo.narHash); - entry.nonFlakeEntries.insert_or_assign(nonFlake.alias, nonEntry); - } - - return entry; -} - bool allowedToWrite(HandleLockFile handle) { return handle == UpdateLockFile || handle == RecreateLockFile; @@ -435,43 +412,50 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } -ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flakeRef, - HandleLockFile handleLockFile, LockFile lockFile = {}, bool topRef = false) +static std::pair updateLocks( + EvalState & state, + const FlakeRef & flakeRef, + HandleLockFile handleLockFile, + const FlakeInputs & oldEntry, + bool topRef) { - Flake flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); + auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - ResolvedFlake deps(flake); + FlakeDep newEntry( + flake.id, + flake.sourceInfo.resolvedRef, + flake.sourceInfo.narHash); - for (auto & nonFlakeInfo : flake.nonFlakeInputs) { - FlakeRef ref = nonFlakeInfo.second; - auto i = lockFile.nonFlakeEntries.find(nonFlakeInfo.first); - if (i != lockFile.nonFlakeEntries.end()) { - NonFlake nonFlake = getNonFlake(state, i->second.ref, nonFlakeInfo.first); - if (nonFlake.sourceInfo.narHash != i->second.narHash) - throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); - deps.nonFlakeDeps.push_back(nonFlake); + for (auto & input : flake.nonFlakeInputs) { + auto & id = input.first; + auto & ref = input.second; + auto i = oldEntry.nonFlakeDeps.find(id); + if (i != oldEntry.nonFlakeDeps.end()) { + newEntry.nonFlakeDeps.insert_or_assign(i->first, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update non-flake dependency '%s' in pure mode", nonFlakeInfo.first); - deps.nonFlakeDeps.push_back(getNonFlake(state, nonFlakeInfo.second, nonFlakeInfo.first, allowedToUseRegistries(handleLockFile, false))); + throw Error("cannot update non-flake dependency '%s' in pure mode", id); + auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); + newEntry.nonFlakeDeps.insert_or_assign(id, + NonFlakeDep( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash)); } } - for (auto newFlakeRef : flake.inputs) { - auto i = lockFile.flakeEntries.find(newFlakeRef); - if (i != lockFile.flakeEntries.end()) { // Propagate lockFile downwards if possible - ResolvedFlake newResFlake = resolveFlakeFromLockFile(state, i->second.ref, handleLockFile, entryToLockFile(i->second)); - if (newResFlake.flake.sourceInfo.narHash != i->second.narHash) - throw Error("the content hash of flakeref '%s' doesn't match", i->second.ref.to_string()); - deps.flakeDeps.insert_or_assign(newFlakeRef, newResFlake); + for (auto & inputRef : flake.inputs) { + auto i = oldEntry.flakeDeps.find(inputRef); + if (i != oldEntry.flakeDeps.end()) { + newEntry.flakeDeps.insert_or_assign(inputRef, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update flake dependency '%s' in pure mode", newFlakeRef.to_string()); - deps.flakeDeps.insert_or_assign(newFlakeRef, resolveFlakeFromLockFile(state, newFlakeRef, handleLockFile)); + throw Error("cannot update flake dependency '%s' in pure mode", inputRef); + newEntry.flakeDeps.insert_or_assign(inputRef, + updateLocks(state, inputRef, handleLockFile, {}, false).second); } } - return deps; + return {flake, newEntry}; } /* Given a flake reference, recursively fetch it and its dependencies. @@ -479,7 +463,8 @@ ResolvedFlake resolveFlakeFromLockFile(EvalState & state, const FlakeRef & flake */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { - Flake flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + LockFile oldLockFile; if (!recreateLockFile(handleLockFile)) { @@ -490,10 +475,9 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } - LockFile lockFile(oldLockFile); - - ResolvedFlake resFlake = resolveFlakeFromLockFile(state, topRef, handleLockFile, lockFile, true); - lockFile = entryToLockFile(dependenciesToFlakeEntry(resFlake)); + // FIXME: get rid of duplicate getFlake call + LockFile lockFile(updateLocks( + state, topRef, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { @@ -509,7 +493,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc warn("using updated lockfile without writing it to file"); } - return resFlake; + return ResolvedFlake(std::move(flake), std::move(lockFile)); } void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) @@ -520,7 +504,9 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) { auto & path = sourceInfo.storePath; - state.store->isValidPath(path); + assert(state.store->isValidPath(path)); + // FIXME: turn into fetchGit etc. + // FIXME: check narHash. mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); if (sourceInfo.resolvedRef.rev) { @@ -539,42 +525,74 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); } -void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v) +/* Helper primop to make callFlake (below) fetch/call its inputs + lazily. Note that this primop cannot be called by user code since + it doesn't appear in 'builtins'. */ +static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyFlake = (FlakeDep *) args[0]->attrs; + auto flake = getFlake(state, lazyFlake->ref, false); + callFlake(state, flake, *lazyFlake, v); +} + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v) { // Construct the resulting attrset '{description, outputs, // ...}'. This attrset is passed lazily as an argument to 'outputs'. - state.mkAttrs(v, resFlake.flakeDeps.size() + resFlake.nonFlakeDeps.size() + 8); + state.mkAttrs(v, + inputs.flakeDeps.size() + + inputs.nonFlakeDeps.size() + 8); - for (auto info : resFlake.flakeDeps) { - const ResolvedFlake newResFlake = info.second; - auto vFlake = state.allocAttr(v, newResFlake.flake.id); - callFlake(state, newResFlake, *vFlake); + for (auto & dep : inputs.flakeDeps) { + auto vFlake = state.allocAttr(v, dep.second.id); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new FlakeDep(dep.second); // evil! also inefficient + mkApp(*vFlake, *vPrimOp, *vArg); } - for (const NonFlake nonFlake : resFlake.nonFlakeDeps) { - auto vNonFlake = state.allocAttr(v, nonFlake.alias); + for (auto & dep : inputs.nonFlakeDeps) { + auto vNonFlake = state.allocAttr(v, dep.first); state.mkAttrs(*vNonFlake, 8); - state.store->isValidPath(nonFlake.sourceInfo.storePath); + auto nonFlake = getNonFlake(state, dep.second.ref, dep.first); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + mkString(*state.allocAttr(*vNonFlake, state.sOutPath), nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); } - mkString(*state.allocAttr(v, state.sDescription), resFlake.flake.description); + mkString(*state.allocAttr(v, state.sDescription), flake.description); - emitSourceInfoAttrs(state, resFlake.flake.sourceInfo, v); + emitSourceInfoAttrs(state, flake.sourceInfo, v); auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); - mkApp(*vOutputs, *resFlake.flake.vOutputs, v); + mkApp(*vOutputs, *flake.vOutputs, v); v.attrs->push_back(Attr(state.symbols.create("self"), &v)); v.attrs->sort(); } +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v) +{ + callFlake(state, resFlake.flake, resFlake.lockFile, v); +} + // This function is exposed to be used in nix files. static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 82b0973f6..692fa744d 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -2,6 +2,7 @@ #include "flakeref.hh" #include +#include namespace nix { @@ -19,51 +20,12 @@ struct FlakeRegistry std::map entries; }; -struct LockFile -{ - struct NonFlakeEntry - { - FlakeRef ref; - Hash narHash; - NonFlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; - - bool operator ==(const NonFlakeEntry & other) const - { - return ref == other.ref && narHash == other.narHash; - } - }; - - struct FlakeEntry - { - FlakeRef ref; - Hash narHash; - std::map flakeEntries; - std::map nonFlakeEntries; - FlakeEntry(const FlakeRef & flakeRef, const Hash & hash) : ref(flakeRef), narHash(hash) {}; - - bool operator ==(const FlakeEntry & other) const - { - return - ref == other.ref - && narHash == other.narHash - && flakeEntries == other.flakeEntries - && nonFlakeEntries == other.nonFlakeEntries; - } - }; - - std::map flakeEntries; - std::map nonFlakeEntries; - - bool operator ==(const LockFile & other) const - { - return - flakeEntries == other.flakeEntries - && nonFlakeEntries == other.nonFlakeEntries; - } -}; - typedef std::vector> Registries; +std::shared_ptr readRegistry(const Path &); + +void writeRegistry(const FlakeRegistry &, const Path &); + Path getUserRegistryPath(); enum HandleLockFile : unsigned int @@ -75,9 +37,80 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -std::shared_ptr readRegistry(const Path &); +struct NonFlakeDep +{ + FlakeRef ref; + Hash narHash; -void writeRegistry(const FlakeRegistry &, const Path &); + NonFlakeDep(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) {}; + + NonFlakeDep(const nlohmann::json & json); + + bool operator ==(const NonFlakeDep & other) const + { + return ref == other.ref && narHash == other.narHash; + } + + nlohmann::json toJson() const; +}; + +struct FlakeDep; + +struct FlakeInputs +{ + std::map flakeDeps; + std::map nonFlakeDeps; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +struct FlakeDep : FlakeInputs +{ + FlakeId id; + FlakeRef ref; + Hash narHash; + + FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : id(id), ref(flakeRef), narHash(narHash) {}; + + FlakeDep(const nlohmann::json & json); + + bool operator ==(const FlakeDep & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + nlohmann::json toJson() const; +}; + +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeDep && dep) + { + flakeDeps = std::move(dep.flakeDeps); + nonFlakeDeps = std::move(dep.nonFlakeDeps); + } + + nlohmann::json toJson() const; +}; struct SourceInfo { @@ -129,14 +162,21 @@ Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); struct ResolvedFlake { Flake flake; - std::map flakeDeps; // The key in this map, is the originalRef as written in flake.nix - std::vector nonFlakeDeps; - ResolvedFlake(const Flake & flake) : flake(flake) {} + LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) + : flake(flake), lockFile(lockFile) {} }; ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); -void callFlake(EvalState & state, const ResolvedFlake & resFlake, Value & v); +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v); + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v); void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8d6716391..d229c7512 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -136,6 +136,7 @@ static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) return j; } +#if 0 // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { @@ -173,6 +174,7 @@ struct CmdFlakeDeps : FlakeCommand } } }; +#endif struct CmdFlakeUpdate : FlakeCommand { @@ -232,6 +234,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON if (json) { auto json = flakeToJson(flake); +#if 0 auto state = getEvalState(); auto vFlake = state->allocValue(); @@ -254,6 +257,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON }); json["outputs"] = std::move(outputs); +#endif std::cout << json.dump() << std::endl; } else @@ -518,7 +522,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command , make_ref() , make_ref() , make_ref() - , make_ref() + //, make_ref() , make_ref() , make_ref() , make_ref() diff --git a/src/nix/installables.cc b/src/nix/installables.cc index b6f05b314..86b4a9b93 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -190,6 +190,7 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const flake::ResolvedFlake & resFlake) { +#if 0 if (std::get_if(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ @@ -224,6 +225,7 @@ void makeFlakeClosureGCRoot(Store & store, debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); replaceSymlink(closurePath, symlink); store.addIndirectRoot(symlink); +#endif } struct InstallableFlake : InstallableValue diff --git a/src/nlohmann/json_fwd.hpp b/src/nlohmann/json_fwd.hpp new file mode 100644 index 000000000..ae6e4c64f --- /dev/null +++ b/src/nlohmann/json_fwd.hpp @@ -0,0 +1,10 @@ +#pragma once + +namespace nlohmann { + +struct json : basic_json<> +{ + using basic_json<>::basic_json; +}; + +} diff --git a/tests/flakes.sh b/tests/flakes.sh index 998abfd09..29845e5ed 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -191,6 +191,8 @@ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth # Check whether it saved the lockfile [[ ! (-z $(git -C $flake3Dir diff master)) ]] +git -C $flake3Dir commit -m 'Add lockfile' + # Unsupported epochs should be an error. sed -i $flake3Dir/flake.nix -e s/201906/201909/ nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported epoch' @@ -241,3 +243,12 @@ git -C $flake3Dir commit -m 'Add nonFlakeInputs' # Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth + +# Check whether flake input fetching is lazy: flake3:sth does not +# depend on flake2, so this shouldn't fail. +rm -rf $TEST_HOME/.cache +clearStore +mv $flake2Dir $flake2Dir.tmp +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy) +mv $flake2Dir.tmp $flake2Dir From 278114d559109199ff8e6f23b6700ab7909f5320 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 19:45:16 +0200 Subject: [PATCH 0291/2548] Fix GC closure generation --- src/libexpr/primops/flake.cc | 17 +++++++++-------- src/libexpr/primops/flake.hh | 26 +++++++++++++++++--------- src/nix/installables.cc | 21 +++++++++++++-------- tests/flakes.sh | 1 + 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 189663e51..d0405a377 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -50,7 +50,7 @@ void writeRegistry(const FlakeRegistry & registry, const Path & path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -NonFlakeDep::NonFlakeDep(const nlohmann::json & json) +AbstractDep::AbstractDep(const nlohmann::json & json) : ref(json["uri"]) , narHash(Hash((std::string) json["narHash"])) { @@ -58,7 +58,7 @@ NonFlakeDep::NonFlakeDep(const nlohmann::json & json) throw Error("lockfile contains mutable flakeref '%s'", ref); } -nlohmann::json NonFlakeDep::toJson() const +nlohmann::json AbstractDep::toJson() const { nlohmann::json json; json["uri"] = ref.to_string(); @@ -66,22 +66,23 @@ nlohmann::json NonFlakeDep::toJson() const return json; } +Path AbstractDep::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + FlakeDep::FlakeDep(const nlohmann::json & json) : FlakeInputs(json) + , AbstractDep(json) , id(json["id"]) - , ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) { - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); } nlohmann::json FlakeDep::toJson() const { auto json = FlakeInputs::toJson(); + json.update(AbstractDep::toJson()); json["id"] = id; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); return json; } diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 692fa744d..933bc2593 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -1,3 +1,5 @@ +#pragma once + #include "types.hh" #include "flakeref.hh" @@ -8,6 +10,7 @@ namespace nix { struct Value; class EvalState; +class Store; namespace flake { @@ -37,22 +40,29 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -struct NonFlakeDep +struct AbstractDep { FlakeRef ref; Hash narHash; - NonFlakeDep(const FlakeRef & flakeRef, const Hash & narHash) + AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) : ref(flakeRef), narHash(narHash) {}; - NonFlakeDep(const nlohmann::json & json); + AbstractDep(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +struct NonFlakeDep : AbstractDep +{ + using AbstractDep::AbstractDep; bool operator ==(const NonFlakeDep & other) const { return ref == other.ref && narHash == other.narHash; } - - nlohmann::json toJson() const; }; struct FlakeDep; @@ -68,14 +78,12 @@ struct FlakeInputs nlohmann::json toJson() const; }; -struct FlakeDep : FlakeInputs +struct FlakeDep : FlakeInputs, AbstractDep { FlakeId id; - FlakeRef ref; - Hash narHash; FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : id(id), ref(flakeRef), narHash(narHash) {}; + : AbstractDep(flakeRef, narHash), id(id) {}; FlakeDep(const nlohmann::json & json); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 86b4a9b93..c44a37f1e 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -190,22 +190,28 @@ void makeFlakeClosureGCRoot(Store & store, const FlakeRef & origFlakeRef, const flake::ResolvedFlake & resFlake) { -#if 0 if (std::get_if(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ PathSet closure; - std::queue> queue; - queue.push(resFlake); + assert(store.isValidPath(resFlake.flake.sourceInfo.storePath)); + closure.insert(resFlake.flake.sourceInfo.storePath); + + std::queue> queue; + queue.push(resFlake.lockFile); while (!queue.empty()) { - const flake::ResolvedFlake & flake = queue.front(); + const flake::FlakeInputs & flake = queue.front(); queue.pop(); - if (!std::get_if(&flake.flake.sourceInfo.resolvedRef.data)) - closure.insert(flake.flake.sourceInfo.storePath); - for (const auto & dep : flake.flakeDeps) + /* Note: due to lazy fetching, these paths might not exist + yet. */ + for (auto & dep : flake.flakeDeps) { + closure.insert(dep.second.computeStorePath(store)); queue.push(dep.second); + } + for (auto & dep : flake.nonFlakeDeps) + closure.insert(dep.second.computeStorePath(store)); } if (closure.empty()) return; @@ -225,7 +231,6 @@ void makeFlakeClosureGCRoot(Store & store, debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); replaceSymlink(closurePath, symlink); store.addIndirectRoot(symlink); -#endif } struct InstallableFlake : InstallableValue diff --git a/tests/flakes.sh b/tests/flakes.sh index 29845e5ed..f44b9509f 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -204,6 +204,7 @@ nix flake list --flake-registry file://$registry --tarball-ttl 0 | grep -q flake mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. +# FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar mv $flake1Dir $flake1Dir.tmp From 9e99b5205c4035753106448241ae44e7447f019c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:01:21 +0200 Subject: [PATCH 0292/2548] Move LockFile and related types to a separate file --- src/libexpr/primops/flake.cc | 103 +----------------------------- src/libexpr/primops/flake.hh | 85 +----------------------- src/libexpr/primops/lockfile.cc | 104 ++++++++++++++++++++++++++++++ src/libexpr/primops/lockfile.hh | 110 ++++++++++++++++++++++++++++++++ 4 files changed, 218 insertions(+), 184 deletions(-) create mode 100644 src/libexpr/primops/lockfile.cc create mode 100644 src/libexpr/primops/lockfile.hh diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index d0405a377..b38971c36 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" #include "fetchGit.hh" @@ -50,104 +51,6 @@ void writeRegistry(const FlakeRegistry & registry, const Path & path) writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } -AbstractDep::AbstractDep(const nlohmann::json & json) - : ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) -{ - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); -} - -nlohmann::json AbstractDep::toJson() const -{ - nlohmann::json json; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); - return json; -} - -Path AbstractDep::computeStorePath(Store & store) const -{ - return store.makeFixedOutputPath(true, narHash, "source"); -} - -FlakeDep::FlakeDep(const nlohmann::json & json) - : FlakeInputs(json) - , AbstractDep(json) - , id(json["id"]) -{ -} - -nlohmann::json FlakeDep::toJson() const -{ - auto json = FlakeInputs::toJson(); - json.update(AbstractDep::toJson()); - json["id"] = id; - return json; -} - -FlakeInputs::FlakeInputs(const nlohmann::json & json) -{ - auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) - nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); - - auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); -} - -nlohmann::json FlakeInputs::toJson() const -{ - nlohmann::json json; - { - auto j = nlohmann::json::object(); - for (auto & i : nonFlakeDeps) - j[i.first] = i.second.toJson(); - json["nonFlakeInputs"] = std::move(j); - } - { - auto j = nlohmann::json::object(); - for (auto & i : flakeDeps) - j[i.first.to_string()] = i.second.toJson(); - json["inputs"] = std::move(j); - } - return json; -} - -nlohmann::json LockFile::toJson() const -{ - auto json = FlakeInputs::toJson(); - json["version"] = 2; - return json; -} - -LockFile readLockFile(const Path & path) -{ - if (pathExists(path)) { - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 2) - throw Error("lock file '%s' has unsupported version %d", path, version); - - return LockFile(json); - } else - return LockFile(); -} - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJson().dump(4); // '4' = indentation in json file - return stream; -} - -void writeLockFile(const LockFile & lockFile, const Path & path) -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", lockFile)); -} - Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -471,7 +374,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!recreateLockFile(handleLockFile)) { // If recreateLockFile, start with an empty lockfile // FIXME: symlink attack - oldLockFile = readLockFile( + oldLockFile = LockFile::read( state.store->toRealPath(flake.sourceInfo.storePath) + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } @@ -483,7 +386,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { - writeLockFile(lockFile, refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. runProgram("git", true, { "-C", refData->path, "add", diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index 933bc2593..bbf35da02 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -2,15 +2,12 @@ #include "types.hh" #include "flakeref.hh" - -#include -#include +#include "lockfile.hh" namespace nix { struct Value; class EvalState; -class Store; namespace flake { @@ -40,86 +37,6 @@ enum HandleLockFile : unsigned int , UseNewLockFile // `RecreateLockFile` without writing to file }; -struct AbstractDep -{ - FlakeRef ref; - Hash narHash; - - AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) - : ref(flakeRef), narHash(narHash) {}; - - AbstractDep(const nlohmann::json & json); - - nlohmann::json toJson() const; - - Path computeStorePath(Store & store) const; -}; - -struct NonFlakeDep : AbstractDep -{ - using AbstractDep::AbstractDep; - - bool operator ==(const NonFlakeDep & other) const - { - return ref == other.ref && narHash == other.narHash; - } -}; - -struct FlakeDep; - -struct FlakeInputs -{ - std::map flakeDeps; - std::map nonFlakeDeps; - - FlakeInputs() {}; - FlakeInputs(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -struct FlakeDep : FlakeInputs, AbstractDep -{ - FlakeId id; - - FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractDep(flakeRef, narHash), id(id) {}; - - FlakeDep(const nlohmann::json & json); - - bool operator ==(const FlakeDep & other) const - { - return - id == other.id - && ref == other.ref - && narHash == other.narHash - && flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; - } - - nlohmann::json toJson() const; -}; - -struct LockFile : FlakeInputs -{ - bool operator ==(const LockFile & other) const - { - return - flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; - } - - LockFile() {} - LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeDep && dep) - { - flakeDeps = std::move(dep.flakeDeps); - nonFlakeDeps = std::move(dep.nonFlakeDeps); - } - - nlohmann::json toJson() const; -}; - struct SourceInfo { // Immutable flakeref that this source tree was obtained from. diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc new file mode 100644 index 000000000..ea0a93510 --- /dev/null +++ b/src/libexpr/primops/lockfile.cc @@ -0,0 +1,104 @@ +#include "lockfile.hh" +#include "store-api.hh" + +namespace nix::flake { + +AbstractDep::AbstractDep(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json AbstractDep::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +Path AbstractDep::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + +FlakeDep::FlakeDep(const nlohmann::json & json) + : FlakeInputs(json) + , AbstractDep(json) + , id(json["id"]) +{ +} + +nlohmann::json FlakeDep::toJson() const +{ + auto json = FlakeInputs::toJson(); + json.update(AbstractDep::toJson()); + json["id"] = id; + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ + auto nonFlakeInputs = json["nonFlakeInputs"]; + for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) + nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); + + auto inputs = json["inputs"]; + for (auto i = inputs.begin(); i != inputs.end(); ++i) + flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); +} + +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeDeps) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeDeps) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; +} + +LockFile LockFile::read(const Path & path) +{ + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); + + return LockFile(json); + } else + return LockFile(); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(4); // '4' = indentation in json file + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +} diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh new file mode 100644 index 000000000..fee15c1da --- /dev/null +++ b/src/libexpr/primops/lockfile.hh @@ -0,0 +1,110 @@ +#pragma once + +#include "flakeref.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::flake { + +/* Common lock file information about a flake input, namely the + immutable ref and the NAR hash. */ +struct AbstractDep +{ + FlakeRef ref; + Hash narHash; + + AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) + { + assert(ref.isImmutable()); + }; + + AbstractDep(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +/* Lock file information about a non-flake input. */ +struct NonFlakeDep : AbstractDep +{ + using AbstractDep::AbstractDep; + + bool operator ==(const NonFlakeDep & other) const + { + return ref == other.ref && narHash == other.narHash; + } +}; + +struct FlakeDep; + +/* Lock file information about the dependencies of a flake. */ +struct FlakeInputs +{ + std::map flakeDeps; + std::map nonFlakeDeps; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +/* Lock file information about a flake input. */ +struct FlakeDep : FlakeInputs, AbstractDep +{ + FlakeId id; + + FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractDep(flakeRef, narHash), id(id) {}; + + FlakeDep(const nlohmann::json & json); + + bool operator ==(const FlakeDep & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + nlohmann::json toJson() const; +}; + +/* An entire lock file. Note that this cannot be a FlakeDep for the + top-level flake, because then the lock file would need to contain + the hash of the top-level flake, but committing the lock file + would invalidate that hash. */ +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeDeps == other.flakeDeps + && nonFlakeDeps == other.nonFlakeDeps; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeDep && dep) + { + flakeDeps = std::move(dep.flakeDeps); + nonFlakeDeps = std::move(dep.nonFlakeDeps); + } + + nlohmann::json toJson() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; +}; + +} + From 5fe7be2409966d673d59d049c3fc6e7710d03b53 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:08:13 +0200 Subject: [PATCH 0293/2548] Rename dep -> input Also use nlohmann::json range-based for. --- src/libexpr/primops/flake.cc | 34 ++++++++++++------------- src/libexpr/primops/lockfile.cc | 28 ++++++++++----------- src/libexpr/primops/lockfile.hh | 44 ++++++++++++++++----------------- src/nix/installables.cc | 4 +-- 4 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index b38971c36..6919bc66b 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -316,7 +316,7 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } -static std::pair updateLocks( +static std::pair updateLocks( EvalState & state, const FlakeRef & flakeRef, HandleLockFile handleLockFile, @@ -325,7 +325,7 @@ static std::pair updateLocks( { auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - FlakeDep newEntry( + FlakeInput newEntry( flake.id, flake.sourceInfo.resolvedRef, flake.sourceInfo.narHash); @@ -333,28 +333,28 @@ static std::pair updateLocks( for (auto & input : flake.nonFlakeInputs) { auto & id = input.first; auto & ref = input.second; - auto i = oldEntry.nonFlakeDeps.find(id); - if (i != oldEntry.nonFlakeDeps.end()) { - newEntry.nonFlakeDeps.insert_or_assign(i->first, i->second); + auto i = oldEntry.nonFlakeInputs.find(id); + if (i != oldEntry.nonFlakeInputs.end()) { + newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", id); auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); - newEntry.nonFlakeDeps.insert_or_assign(id, - NonFlakeDep( + newEntry.nonFlakeInputs.insert_or_assign(id, + NonFlakeInput( nonFlake.sourceInfo.resolvedRef, nonFlake.sourceInfo.narHash)); } } for (auto & inputRef : flake.inputs) { - auto i = oldEntry.flakeDeps.find(inputRef); - if (i != oldEntry.flakeDeps.end()) { - newEntry.flakeDeps.insert_or_assign(inputRef, i->second); + auto i = oldEntry.flakeInputs.find(inputRef); + if (i != oldEntry.flakeInputs.end()) { + newEntry.flakeInputs.insert_or_assign(inputRef, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake dependency '%s' in pure mode", inputRef); - newEntry.flakeDeps.insert_or_assign(inputRef, + newEntry.flakeInputs.insert_or_assign(inputRef, updateLocks(state, inputRef, handleLockFile, {}, false).second); } } @@ -434,7 +434,7 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo it doesn't appear in 'builtins'. */ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - auto lazyFlake = (FlakeDep *) args[0]->attrs; + auto lazyFlake = (FlakeInput *) args[0]->attrs; auto flake = getFlake(state, lazyFlake->ref, false); callFlake(state, flake, *lazyFlake, v); } @@ -448,10 +448,10 @@ void callFlake(EvalState & state, // ...}'. This attrset is passed lazily as an argument to 'outputs'. state.mkAttrs(v, - inputs.flakeDeps.size() + - inputs.nonFlakeDeps.size() + 8); + inputs.flakeInputs.size() + + inputs.nonFlakeInputs.size() + 8); - for (auto & dep : inputs.flakeDeps) { + for (auto & dep : inputs.flakeInputs) { auto vFlake = state.allocAttr(v, dep.second.id); auto vPrimOp = state.allocValue(); static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); @@ -460,11 +460,11 @@ void callFlake(EvalState & state, auto vArg = state.allocValue(); vArg->type = tNull; // FIXME: leak - vArg->attrs = (Bindings *) new FlakeDep(dep.second); // evil! also inefficient + vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient mkApp(*vFlake, *vPrimOp, *vArg); } - for (auto & dep : inputs.nonFlakeDeps) { + for (auto & dep : inputs.nonFlakeInputs) { auto vNonFlake = state.allocAttr(v, dep.first); state.mkAttrs(*vNonFlake, 8); diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc index ea0a93510..97c748c66 100644 --- a/src/libexpr/primops/lockfile.cc +++ b/src/libexpr/primops/lockfile.cc @@ -3,7 +3,7 @@ namespace nix::flake { -AbstractDep::AbstractDep(const nlohmann::json & json) +AbstractInput::AbstractInput(const nlohmann::json & json) : ref(json["uri"]) , narHash(Hash((std::string) json["narHash"])) { @@ -11,7 +11,7 @@ AbstractDep::AbstractDep(const nlohmann::json & json) throw Error("lockfile contains mutable flakeref '%s'", ref); } -nlohmann::json AbstractDep::toJson() const +nlohmann::json AbstractInput::toJson() const { nlohmann::json json; json["uri"] = ref.to_string(); @@ -19,35 +19,33 @@ nlohmann::json AbstractDep::toJson() const return json; } -Path AbstractDep::computeStorePath(Store & store) const +Path AbstractInput::computeStorePath(Store & store) const { return store.makeFixedOutputPath(true, narHash, "source"); } -FlakeDep::FlakeDep(const nlohmann::json & json) +FlakeInput::FlakeInput(const nlohmann::json & json) : FlakeInputs(json) - , AbstractDep(json) + , AbstractInput(json) , id(json["id"]) { } -nlohmann::json FlakeDep::toJson() const +nlohmann::json FlakeInput::toJson() const { auto json = FlakeInputs::toJson(); - json.update(AbstractDep::toJson()); + json.update(AbstractInput::toJson()); json["id"] = id; return json; } FlakeInputs::FlakeInputs(const nlohmann::json & json) { - auto nonFlakeInputs = json["nonFlakeInputs"]; - for (auto i = nonFlakeInputs.begin(); i != nonFlakeInputs.end(); ++i) - nonFlakeDeps.insert_or_assign(i.key(), NonFlakeDep(*i)); + for (auto & i : json["nonFlakeInputs"].items()) + nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); - auto inputs = json["inputs"]; - for (auto i = inputs.begin(); i != inputs.end(); ++i) - flakeDeps.insert_or_assign(i.key(), FlakeDep(*i)); + for (auto & i : json["inputs"].items()) + flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); } nlohmann::json FlakeInputs::toJson() const @@ -55,13 +53,13 @@ nlohmann::json FlakeInputs::toJson() const nlohmann::json json; { auto j = nlohmann::json::object(); - for (auto & i : nonFlakeDeps) + for (auto & i : nonFlakeInputs) j[i.first] = i.second.toJson(); json["nonFlakeInputs"] = std::move(j); } { auto j = nlohmann::json::object(); - for (auto & i : flakeDeps) + for (auto & i : flakeInputs) j[i.first.to_string()] = i.second.toJson(); json["inputs"] = std::move(j); } diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh index fee15c1da..f2e598528 100644 --- a/src/libexpr/primops/lockfile.hh +++ b/src/libexpr/primops/lockfile.hh @@ -12,18 +12,18 @@ namespace nix::flake { /* Common lock file information about a flake input, namely the immutable ref and the NAR hash. */ -struct AbstractDep +struct AbstractInput { FlakeRef ref; Hash narHash; - AbstractDep(const FlakeRef & flakeRef, const Hash & narHash) + AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) : ref(flakeRef), narHash(narHash) { assert(ref.isImmutable()); }; - AbstractDep(const nlohmann::json & json); + AbstractInput(const nlohmann::json & json); nlohmann::json toJson() const; @@ -31,23 +31,23 @@ struct AbstractDep }; /* Lock file information about a non-flake input. */ -struct NonFlakeDep : AbstractDep +struct NonFlakeInput : AbstractInput { - using AbstractDep::AbstractDep; + using AbstractInput::AbstractInput; - bool operator ==(const NonFlakeDep & other) const + bool operator ==(const NonFlakeInput & other) const { return ref == other.ref && narHash == other.narHash; } }; -struct FlakeDep; +struct FlakeInput; /* Lock file information about the dependencies of a flake. */ struct FlakeInputs { - std::map flakeDeps; - std::map nonFlakeDeps; + std::map flakeInputs; + std::map nonFlakeInputs; FlakeInputs() {}; FlakeInputs(const nlohmann::json & json); @@ -56,29 +56,29 @@ struct FlakeInputs }; /* Lock file information about a flake input. */ -struct FlakeDep : FlakeInputs, AbstractDep +struct FlakeInput : FlakeInputs, AbstractInput { FlakeId id; - FlakeDep(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractDep(flakeRef, narHash), id(id) {}; + FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractInput(flakeRef, narHash), id(id) {}; - FlakeDep(const nlohmann::json & json); + FlakeInput(const nlohmann::json & json); - bool operator ==(const FlakeDep & other) const + bool operator ==(const FlakeInput & other) const { return id == other.id && ref == other.ref && narHash == other.narHash - && flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; + && flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; } nlohmann::json toJson() const; }; -/* An entire lock file. Note that this cannot be a FlakeDep for the +/* An entire lock file. Note that this cannot be a FlakeInput for the top-level flake, because then the lock file would need to contain the hash of the top-level flake, but committing the lock file would invalidate that hash. */ @@ -87,16 +87,16 @@ struct LockFile : FlakeInputs bool operator ==(const LockFile & other) const { return - flakeDeps == other.flakeDeps - && nonFlakeDeps == other.nonFlakeDeps; + flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; } LockFile() {} LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeDep && dep) + LockFile(FlakeInput && dep) { - flakeDeps = std::move(dep.flakeDeps); - nonFlakeDeps = std::move(dep.nonFlakeDeps); + flakeInputs = std::move(dep.flakeInputs); + nonFlakeInputs = std::move(dep.nonFlakeInputs); } nlohmann::json toJson() const; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index c44a37f1e..e7549b57c 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -206,11 +206,11 @@ void makeFlakeClosureGCRoot(Store & store, queue.pop(); /* Note: due to lazy fetching, these paths might not exist yet. */ - for (auto & dep : flake.flakeDeps) { + for (auto & dep : flake.flakeInputs) { closure.insert(dep.second.computeStorePath(store)); queue.push(dep.second); } - for (auto & dep : flake.nonFlakeDeps) + for (auto & dep : flake.nonFlakeInputs) closure.insert(dep.second.computeStorePath(store)); } From 4ec1a9ab40705b651fe4b2ef6186c0a6ae279681 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:33:49 +0200 Subject: [PATCH 0294/2548] Fix test --- tests/flakes.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index f44b9509f..f9f8e5272 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -245,6 +245,8 @@ git -C $flake3Dir commit -m 'Add nonFlakeInputs' # Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth +git -C $flake3Dir commit -m 'Update nonFlakeInputs' + # Check whether flake input fetching is lazy: flake3:sth does not # depend on flake2, so this shouldn't fail. rm -rf $TEST_HOME/.cache From 45b5c606ac44550de14562df4fa99773a81a1015 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:34:08 +0200 Subject: [PATCH 0295/2548] Don't register invalid paths as GC roots Unfortunately this doesn't work. Maybe we should keep separate roots for each path. --- src/nix/installables.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index e7549b57c..ca88ec0da 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -207,11 +207,16 @@ void makeFlakeClosureGCRoot(Store & store, /* Note: due to lazy fetching, these paths might not exist yet. */ for (auto & dep : flake.flakeInputs) { - closure.insert(dep.second.computeStorePath(store)); + auto path = dep.second.computeStorePath(store); + if (store.isValidPath(path)) + closure.insert(path); queue.push(dep.second); } - for (auto & dep : flake.nonFlakeInputs) - closure.insert(dep.second.computeStorePath(store)); + for (auto & dep : flake.nonFlakeInputs) { + auto path = dep.second.computeStorePath(store); + if (store.isValidPath(path)) + closure.insert(path); + } } if (closure.empty()) return; From 1c5067b9a7e1f561bf9e9e84642c495a50ca44a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:34:44 +0200 Subject: [PATCH 0296/2548] Check hash --- src/libexpr/primops/flake.cc | 6 ++++-- src/libexpr/primops/lockfile.hh | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 6919bc66b..52d8df69e 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -409,8 +409,6 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo { auto & path = sourceInfo.storePath; assert(state.store->isValidPath(path)); - // FIXME: turn into fetchGit etc. - // FIXME: check narHash. mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); if (sourceInfo.resolvedRef.rev) { @@ -436,6 +434,10 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V { auto lazyFlake = (FlakeInput *) args[0]->attrs; auto flake = getFlake(state, lazyFlake->ref, false); + + if (flake.sourceInfo.narHash != lazyFlake->narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + callFlake(state, flake, *lazyFlake, v); } diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh index f2e598528..b76124190 100644 --- a/src/libexpr/primops/lockfile.hh +++ b/src/libexpr/primops/lockfile.hh @@ -106,5 +106,7 @@ struct LockFile : FlakeInputs void write(const Path & path) const; }; +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + } From 1e53a07712fba830eb3967cc16894992d5a33922 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:56:13 +0200 Subject: [PATCH 0297/2548] Make non-flake inputs lazy Also add a proper test for non-flake inputs. --- src/libexpr/primops/flake.cc | 44 ++++++++++++++++++++++++------------ src/libexpr/primops/flake.hh | 1 - src/nix/flake.cc | 2 +- tests/flakes.sh | 25 +++++++++++++++++--- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 52d8df69e..f99738db5 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -276,7 +276,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe } // Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias alias, bool impureIsAllowed = false) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); @@ -290,8 +290,6 @@ NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, FlakeAlias al if (state.allowedPaths) state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - nonFlake.alias = alias; - return nonFlake; } @@ -339,7 +337,7 @@ static std::pair updateLocks( } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, ref, id, allowedToUseRegistries(handleLockFile, false)); + auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); newEntry.nonFlakeInputs.insert_or_assign(id, NonFlakeInput( nonFlake.sourceInfo.resolvedRef, @@ -441,6 +439,25 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V callFlake(state, flake, *lazyFlake, v); } +static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + + if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + + state.mkAttrs(v, 8); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + + mkString(*state.allocAttr(v, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); +} + void callFlake(EvalState & state, const Flake & flake, const FlakeInputs & inputs, @@ -468,16 +485,15 @@ void callFlake(EvalState & state, for (auto & dep : inputs.nonFlakeInputs) { auto vNonFlake = state.allocAttr(v, dep.first); - state.mkAttrs(*vNonFlake, 8); - - auto nonFlake = getNonFlake(state, dep.second.ref, dep.first); - - assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); - - mkString(*state.allocAttr(*vNonFlake, state.sOutPath), - nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - - emitSourceInfoAttrs(state, nonFlake.sourceInfo, *vNonFlake); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient + mkApp(*vNonFlake, *vPrimOp, *vArg); } mkString(*state.allocAttr(v, state.sDescription), flake.description); diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh index bbf35da02..b8d0da252 100644 --- a/src/libexpr/primops/flake.hh +++ b/src/libexpr/primops/flake.hh @@ -75,7 +75,6 @@ struct Flake struct NonFlake { - FlakeAlias alias; FlakeRef originalRef; SourceInfo sourceInfo; NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d229c7512..653154aaa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -122,6 +122,7 @@ static nlohmann::json flakeToJson(const Flake & flake) return j; } +#if 0 static void printNonFlakeInfo(const NonFlake & nonFlake) { std::cout << fmt("ID: %s\n", nonFlake.alias); @@ -136,7 +137,6 @@ static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) return j; } -#if 0 // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { diff --git a/tests/flakes.sh b/tests/flakes.sh index f9f8e5272..1cd8259b9 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -83,7 +83,7 @@ git -C $flake3Dir add flake.nix git -C $flake3Dir commit -m 'Initial' cat > $nonFlakeDir/README.md < $flake3Dir/flake.nix < \$out + ''; + }; }; } EOF -git -C $flake3Dir add flake.nix +cp ./config.nix $flake3Dir + +git -C $flake3Dir add flake.nix config.nix git -C $flake3Dir commit -m 'Add nonFlakeInputs' -# Check whether `nix build` works with a lockfile which is missing a nonFlakeInputs +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth git -C $flake3Dir commit -m 'Update nonFlakeInputs' +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + # Check whether flake input fetching is lazy: flake3:sth does not # depend on flake2, so this shouldn't fail. rm -rf $TEST_HOME/.cache clearStore mv $flake2Dir $flake2Dir.tmp +mv $nonFlakeDir $nonFlakeDir.tmp nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth (! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy) +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord) mv $flake2Dir.tmp $flake2Dir +mv $nonFlakeDir.tmp $nonFlakeDir +nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy flake3:fnord From 4d31cf83f288ed2e4a14f72e99b77859de981bb4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 20:57:18 +0200 Subject: [PATCH 0298/2548] Update flake.lock --- flake.lock | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b85571b61..727d16d73 100644 --- a/flake.lock +++ b/flake.lock @@ -1,10 +1,13 @@ { "inputs": { "nixpkgs": { - "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", - "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" + "id": "nixpkgs", + "inputs": {}, + "narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=", + "nonFlakeInputs": {}, + "uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1" } }, "nonFlakeInputs": {}, - "version": 1 + "version": 2 } From 087530dec40bd5ab23fe2da83fca517bb91d2282 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 21:07:55 +0200 Subject: [PATCH 0299/2548] Add comments --- src/libexpr/primops/flake.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index f99738db5..000a43764 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -314,6 +314,15 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) else assert(false); } +/* Given a flakeref and its subtree of the lockfile, return an updated + subtree of the lockfile. That is, if the 'flake.nix' of the + referenced flake has inputs that don't have a corresponding entry + in the lockfile, they're added to the lockfile; conversely, any + lockfile entries that don't have a corresponding entry in flake.nix + are removed. + + Note that this is lazy: we only recursively fetch inputs that are + not in the lockfile yet. */ static std::pair updateLocks( EvalState & state, const FlakeRef & flakeRef, @@ -360,9 +369,8 @@ static std::pair updateLocks( return {flake, newEntry}; } -/* Given a flake reference, recursively fetch it and its dependencies. - FIXME: this should return a graph of flakes. -*/ +/* Compute an in-memory lockfile for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); From ce225615c3ee08b7b63a8488dbf74ff2598d8d74 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 21:10:53 +0200 Subject: [PATCH 0300/2548] Eliminate duplicate fetching of the top-level flake --- src/libexpr/primops/flake.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 000a43764..734f650f2 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -325,13 +325,11 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) not in the lockfile yet. */ static std::pair updateLocks( EvalState & state, - const FlakeRef & flakeRef, + const Flake & flake, HandleLockFile handleLockFile, const FlakeInputs & oldEntry, bool topRef) { - auto flake = getFlake(state, flakeRef, allowedToUseRegistries(handleLockFile, topRef)); - FlakeInput newEntry( flake.id, flake.sourceInfo.resolvedRef, @@ -362,7 +360,9 @@ static std::pair updateLocks( if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake dependency '%s' in pure mode", inputRef); newEntry.flakeInputs.insert_or_assign(inputRef, - updateLocks(state, inputRef, handleLockFile, {}, false).second); + updateLocks(state, + getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + handleLockFile, {}, false).second); } } @@ -385,9 +385,8 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } - // FIXME: get rid of duplicate getFlake call LockFile lockFile(updateLocks( - state, topRef, handleLockFile, oldLockFile, true).second); + state, flake, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { From 1b057929885fd3f339d4c85b44ad9f10fef7d8a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 Jun 2019 22:35:43 +0200 Subject: [PATCH 0301/2548] Shorter syntax for referencing flake outputs Fixes #2819. --- src/libexpr/primops/flake.cc | 16 +++++++++++++--- tests/flakes.sh | 12 ++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc index 734f650f2..793d6da35 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/primops/flake.cc @@ -468,10 +468,12 @@ static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args void callFlake(EvalState & state, const Flake & flake, const FlakeInputs & inputs, - Value & v) + Value & vRes) { - // Construct the resulting attrset '{description, outputs, - // ...}'. This attrset is passed lazily as an argument to 'outputs'. + // Construct the resulting attrset '{outputs, ...}'. This attrset + // is passed lazily as an argument to the 'outputs' function. + + auto & v = *state.allocValue(); state.mkAttrs(v, inputs.flakeInputs.size() + @@ -513,6 +515,14 @@ void callFlake(EvalState & state, v.attrs->push_back(Attr(state.symbols.create("self"), &v)); v.attrs->sort(); + + /* For convenience, put the outputs directly in the result, so you + can refer to an output of an input as 'inputs.foo.bar' rather + than 'inputs.foo.outputs.bar'. */ + auto v2 = *state.allocValue(); + state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + + state.callFunction(v2, v, vRes, noPos); } void callFlake(EvalState & state, diff --git a/tests/flakes.sh b/tests/flakes.sh index 1cd8259b9..c380b405b 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -55,7 +55,7 @@ cat > $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Wed, 5 Jun 2019 00:40:45 -0400 Subject: [PATCH 0302/2548] Handle SIGWINCH in main thread For the SIGWINCH signal to be caught, it needs to be set in sigaction on the main thread. Previously, this was broken, and updateWindowSize was never being called. Tested on macOS 10.14. --- src/libmain/shared.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..f2aeec9e1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,6 +125,9 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + /* Make sure SIGWINCH is handled as well. */ + if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); From 54aff8430c4e7739903f6dbed713cc088e38507f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Jun 2019 16:51:54 +0200 Subject: [PATCH 0303/2548] Move flake-related stuff to src/libexpr/flake --- src/libexpr/eval.cc | 2 +- src/libexpr/{primops => flake}/flake.cc | 2 +- src/libexpr/{primops => flake}/flake.hh | 0 src/libexpr/{primops => flake}/flakeref.cc | 0 src/libexpr/{primops => flake}/flakeref.hh | 0 src/libexpr/{primops => flake}/lockfile.cc | 0 src/libexpr/{primops => flake}/lockfile.hh | 0 src/libexpr/local.mk | 7 ++++++- src/nix/flake.cc | 2 +- src/nix/installables.cc | 2 +- 10 files changed, 10 insertions(+), 5 deletions(-) rename src/libexpr/{primops => flake}/flake.cc (99%) rename src/libexpr/{primops => flake}/flake.hh (100%) rename src/libexpr/{primops => flake}/flakeref.cc (100%) rename src/libexpr/{primops => flake}/flakeref.hh (100%) rename src/libexpr/{primops => flake}/lockfile.cc (100%) rename src/libexpr/{primops => flake}/lockfile.hh (100%) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0f8a105b1..46c622ee8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,7 +7,7 @@ #include "eval-inline.hh" #include "download.hh" #include "json.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/flake/flake.cc similarity index 99% rename from src/libexpr/primops/flake.cc rename to src/libexpr/flake/flake.cc index 793d6da35..bb0543541 100644 --- a/src/libexpr/primops/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -2,7 +2,7 @@ #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" -#include "fetchGit.hh" +#include "primops/fetchGit.hh" #include "download.hh" #include "args.hh" diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/flake/flake.hh similarity index 100% rename from src/libexpr/primops/flake.hh rename to src/libexpr/flake/flake.hh diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/flake/flakeref.cc similarity index 100% rename from src/libexpr/primops/flakeref.cc rename to src/libexpr/flake/flakeref.cc diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/flake/flakeref.hh similarity index 100% rename from src/libexpr/primops/flakeref.hh rename to src/libexpr/flake/flakeref.hh diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/flake/lockfile.cc similarity index 100% rename from src/libexpr/primops/lockfile.cc rename to src/libexpr/flake/lockfile.cc diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/flake/lockfile.hh similarity index 100% rename from src/libexpr/primops/lockfile.hh rename to src/libexpr/flake/lockfile.hh diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ccd5293e4..a9cb6b7b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_LIBS = libutil libstore diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 653154aaa..af1a361b3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,7 +4,7 @@ #include "progress-bar.hh" #include "eval.hh" #include "eval-inline.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ca88ec0da..a85295a09 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,7 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include From 5011a52cf3f291c22d784aff6f5bbf8b99ae186a Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 5 Jun 2019 20:18:47 -0400 Subject: [PATCH 0304/2548] Just enable hack on macOS This is not needed on linux at all! Tried to explain as much as I understand with the problem. --- src/libmain/shared.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f2aeec9e1..a6101342a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,8 +125,14 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); - /* Make sure SIGWINCH is handled as well. */ - if (sigaction(SIGWINCH, &act, 0)) throw SysError("handling SIGWINCH"); +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); From 6644b6099be2d3393206bf1c9c091c888c0a0f57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Jun 2019 22:25:48 +0200 Subject: [PATCH 0305/2548] Add flake evaluation cache This exploits the hermetic nature of flake evaluation to speed up repeated evaluations of a flake output attribute. For example (doing 'nix build' on an already present package): $ time nix build nixpkgs:firefox real 0m1.497s user 0m1.160s sys 0m0.139s $ time nix build nixpkgs:firefox real 0m0.052s user 0m0.038s sys 0m0.007s The cache is ~/.cache/nix/eval-cache-v1.sqlite, which has entries like INSERT INTO Attributes VALUES( X'92a907d4efe933af2a46959b082cdff176aa5bfeb47a98fabd234809a67ab195', 'packages.firefox', 1, '/nix/store/pbalzf8x19hckr8cwdv62rd6g0lqgc38-firefox-67.0.drv /nix/store/g6q0gx0v6xvdnizp8lrcw7c4gdkzana0-firefox-67.0 out'); where the hash 92a9... is a fingerprint over the flake store path and the contents of the lockfile. Because flakes are evaluated in pure mode, this uniquely identifies the evaluation result. --- src/libexpr/flake/eval-cache.cc | 111 ++++++++++++++++++++++++ src/libexpr/flake/eval-cache.hh | 39 +++++++++ src/libexpr/flake/flake.cc | 9 ++ src/libexpr/flake/flake.hh | 6 ++ src/libstore/local-store.cc | 9 +- src/libstore/nar-info-disk-cache.cc | 7 +- src/libstore/sqlite.cc | 25 +++++- src/libstore/sqlite.hh | 6 +- src/nix/installables.cc | 130 +++++++++++++++++++++------- 9 files changed, 293 insertions(+), 49 deletions(-) create mode 100644 src/libexpr/flake/eval-cache.cc create mode 100644 src/libexpr/flake/eval-cache.hh diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc new file mode 100644 index 000000000..fece1a2b5 --- /dev/null +++ b/src/libexpr/flake/eval-cache.cc @@ -0,0 +1,111 @@ +#include "eval-cache.hh" +#include "sqlite.hh" + +#include + +namespace nix::flake { + +static const char * schema = R"sql( + +create table if not exists Fingerprints ( + fingerprint blob primary key not null, + timestamp integer not null +); + +create table if not exists Attributes ( + fingerprint blob not null, + attrPath text not null, + type integer, + value text, + primary key (fingerprint, attrPath), + foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade +); +)sql"; + +struct EvalCache::State +{ + SQLite db; + SQLiteStmt insertFingerprint; + SQLiteStmt insertAttribute; + SQLiteStmt queryAttribute; + std::set fingerprints; +}; + +EvalCache::EvalCache() + : _state(std::make_unique>()) +{ + auto state(_state->lock()); + + Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertFingerprint.create(state->db, + "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)"); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select type, value from Attributes where fingerprint = ? and attrPath = ?"); +} + +enum ValueType { + Derivation = 1, +}; + +void EvalCache::addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv) +{ + auto state(_state->lock()); + + if (state->fingerprints.insert(fingerprint).second) + // FIXME: update timestamp + state->insertFingerprint.use() + (fingerprint.hash, fingerprint.hashSize) + (time(0)).exec(); + + state->insertAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath) + (ValueType::Derivation) + (drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec(); +} + +std::optional EvalCache::getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath) +{ + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath)); + if (!queryAttribute.next()) return {}; + + // FIXME: handle negative results + + auto type = (ValueType) queryAttribute.getInt(0); + auto s = queryAttribute.getStr(1); + + if (type != ValueType::Derivation) return {}; + + auto ss = tokenizeString>(s, " "); + + debug("evaluation cache hit for '%s'", attrPath); + + return Derivation { ss[0], ss[1], ss[2] }; +} + +EvalCache & EvalCache::singleton() +{ + static std::unique_ptr evalCache(new EvalCache()); + return *evalCache; +} + +} diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh new file mode 100644 index 000000000..03aea142e --- /dev/null +++ b/src/libexpr/flake/eval-cache.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "sync.hh" +#include "flake.hh" + +namespace nix { struct SQLite; struct SQLiteStmt; } + +namespace nix::flake { + +class EvalCache +{ + struct State; + + std::unique_ptr> _state; + + EvalCache(); + +public: + + struct Derivation + { + Path drvPath; + Path outPath; + std::string outputName; + }; + + void addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv); + + std::optional getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath); + + static EvalCache & singleton(); +}; + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index bb0543541..0018a0d07 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -601,4 +601,13 @@ const Registries EvalState::getFlakeRegistries() return registries; } +Fingerprint ResolvedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lockfile and + // we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%s", flake.sourceInfo.storePath, lockFile)); +} + } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b8d0da252..81b6541f0 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -83,12 +83,18 @@ struct NonFlake Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + struct ResolvedFlake { Flake flake; LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) : flake(flake), lockFile(lockFile) {} + + Fingerprint getFingerprint() const; }; ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..f39c73b23 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -294,9 +294,7 @@ void LocalStore::openDB(State & state, bool create) /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -308,11 +306,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 32ad7f2b2..3f6dbbcf5 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..eb1daafc5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open SQLite database '%s'") % path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 115679b84..78e53fa32 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a85295a09..86e601bc4 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -8,6 +8,7 @@ #include "store-api.hh" #include "shared.hh" #include "flake/flake.hh" +#include "flake/eval-cache.hh" #include #include @@ -110,7 +111,7 @@ struct InstallableValue : Installable InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } - Buildables toBuildables() override + virtual std::vector toDerivations() { auto state = cmd.getEvalState(); @@ -118,22 +119,36 @@ struct InstallableValue : Installable Bindings & autoArgs = *cmd.getAutoArgs(*state); - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + std::vector res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + drvInfo.queryDrvPath(), + drvInfo.queryOutPath(), + drvInfo.queryOutputName() + }); + } + + return res; + } + + Buildables toBuildables() override + { Buildables res; PathSet drvPaths; - for (auto & drv : drvs) { - Buildable b{drv.queryDrvPath()}; + for (auto & drv : toDerivations()) { + Buildable b{drv.drvPath}; drvPaths.insert(b.drvPath); - auto outputName = drv.queryOutputName(); + auto outputName = drv.outputName; if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); - b.outputs.emplace(outputName, drv.queryOutPath()); + b.outputs.emplace(outputName, drv.outPath); res.push_back(std::move(b)); } @@ -254,12 +269,30 @@ struct InstallableFlake : InstallableValue std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } - Value * toValue(EvalState & state) override + std::vector getActualAttrPaths() + { + std::vector res; + + if (searchPackages) { + // As a convenience, look for the attribute in + // 'outputs.packages'. + res.push_back("packages." + *attrPaths.begin()); + + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + res.push_back("legacyPackages." + *attrPaths.begin()); + } + + for (auto & s : attrPaths) + res.push_back(s); + + return res; + } + + Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake) { auto vFlake = state.allocValue(); - auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); - callFlake(state, resFlake, *vFlake); makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); @@ -268,34 +301,67 @@ struct InstallableFlake : InstallableValue state.forceValue(*vOutputs); - auto emptyArgs = state.allocBindings(0); + return vOutputs; + } - if (searchPackages) { - // As a convenience, look for the attribute in - // 'outputs.packages'. - if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("packages"))) { - try { - auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); - state.forceValue(*v); - return v; - } catch (AttrPathNotFound & e) { - } + std::vector toDerivations() override + { + auto state = cmd.getEvalState(); + + auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode()); + + Value * vOutputs = nullptr; + + auto emptyArgs = state->allocBindings(0); + + auto & evalCache = flake::EvalCache::singleton(); + + auto fingerprint = resFlake.getFingerprint(); + + for (auto & attrPath : getActualAttrPaths()) { + auto drv = evalCache.getDerivation(fingerprint, attrPath); + if (drv) { + if (state->store->isValidPath(drv->drvPath)) + return {*drv}; } - // As a temporary hack until Nixpkgs is properly converted - // to provide a clean 'packages' set, look in 'legacyPackages'. - if (auto aPackages = *vOutputs->attrs->get(state.symbols.create("legacyPackages"))) { - try { - auto * v = findAlongAttrPath(state, *attrPaths.begin(), *emptyArgs, *aPackages->value); - state.forceValue(*v); - return v; - } catch (AttrPathNotFound & e) { - } + if (!vOutputs) + vOutputs = getFlakeOutputs(*state, resFlake); + + try { + auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs); + state->forceValue(*v); + + auto drvInfo = getDerivation(*state, *v, false); + if (!drvInfo) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto drv = flake::EvalCache::Derivation{ + drvInfo->queryDrvPath(), + drvInfo->queryOutPath(), + drvInfo->queryOutputName() + }; + + evalCache.addDerivation(fingerprint, attrPath, drv); + + return {drv}; + } catch (AttrPathNotFound & e) { } } - // Otherwise, look for it in 'outputs'. - for (auto & attrPath : attrPaths) { + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); + } + + Value * toValue(EvalState & state) override + { + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + auto vOutputs = getFlakeOutputs(state, resFlake); + + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { try { auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); state.forceValue(*v); From c47d2dac6c7b404714e4c3429f26791790a483f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Jun 2019 22:38:39 +0200 Subject: [PATCH 0306/2548] Disable EvalCache in impure mode --- src/libexpr/flake/eval-cache.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc index fece1a2b5..b32d502f7 100644 --- a/src/libexpr/flake/eval-cache.cc +++ b/src/libexpr/flake/eval-cache.cc @@ -1,5 +1,6 @@ #include "eval-cache.hh" #include "sqlite.hh" +#include "eval.hh" #include @@ -62,6 +63,8 @@ void EvalCache::addDerivation( const std::string & attrPath, const Derivation & drv) { + if (!evalSettings.pureEval) return; + auto state(_state->lock()); if (state->fingerprints.insert(fingerprint).second) @@ -81,6 +84,8 @@ std::optional EvalCache::getDerivation( const Fingerprint & fingerprint, const std::string & attrPath) { + if (!evalSettings.pureEval) return {}; + auto state(_state->lock()); auto queryAttribute(state->queryAttribute.use() From 69b047f4ce8b3c915e13fbe9d6e0b60598a1e786 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Jun 2019 21:32:57 +0200 Subject: [PATCH 0307/2548] writeRegistry(): Write correct version --- src/libexpr/flake/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0018a0d07..8aa989620 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -44,7 +44,7 @@ std::shared_ptr readRegistry(const Path & path) void writeRegistry(const FlakeRegistry & registry, const Path & path) { nlohmann::json json; - json["version"] = 2; + json["version"] = 1; for (auto elem : registry.entries) json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; createDirs(dirOf(path)); From e2d75696857af36232f189f2bee4fccec9b6076f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Jun 2019 21:45:57 +0200 Subject: [PATCH 0308/2548] Add a test for GitHub flakes Fixes #2889. --- flake.lock | 4 +- release.nix | 5 ++ tests/github-flakes.nix | 145 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 tests/github-flakes.nix diff --git a/flake.lock b/flake.lock index 727d16d73..336e3eb86 100644 --- a/flake.lock +++ b/flake.lock @@ -3,9 +3,9 @@ "nixpkgs": { "id": "nixpkgs", "inputs": {}, - "narHash": "sha256-eYtxncIMFVmOHaHBtTdPGcs/AnJqKqA6tHCm0UmPYQU=", + "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", "nonFlakeInputs": {}, - "uri": "github:edolstra/nixpkgs/e9d5882bb861dc48f8d46960e7c820efdbe8f9c1" + "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" } }, "nonFlakeInputs": {}, diff --git a/release.nix b/release.nix index d28c44910..28bf7897f 100644 --- a/release.nix +++ b/release.nix @@ -228,6 +228,11 @@ let nix = build.x86_64-linux; system = "x86_64-linux"; }); + tests.githubFlakes = (import ./tests/github-flakes.nix rec { + inherit nixpkgs; + nix = build.x86_64-linux; system = "x86_64-linux"; + }); + tests.setuid = pkgs.lib.genAttrs ["i686-linux" "x86_64-linux"] (system: diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix new file mode 100644 index 000000000..2345972ce --- /dev/null +++ b/tests/github-flakes.nix @@ -0,0 +1,145 @@ +{ nixpkgs, system, nix }: + +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + +let + + # Generate a fake root CA and a fake github.com certificate. + cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } + '' + mkdir -p $out + + openssl genrsa -out ca.key 2048 + openssl req -new -x509 -days 36500 -key ca.key \ + -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt + + openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ + -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr + openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:raw.githubusercontent.com") \ + -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt + ''; + + registry = pkgs.writeTextFile { + name = "registry"; + text = '' + { + "flakes": { + "nixpkgs": { + "uri": "github:NixOS/nixpkgs" + } + }, + "version": 1 + } + ''; + destination = "/flake-registry.json"; + }; + + tarball = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir $out + dir=NixOS-nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModified}.${builtins.substring 12 2 nixpkgs.lastModified} -- + tar cfz $out/${nixpkgs.rev} $dir + ln -s ${nixpkgs.rev} $out/master + ''; + +in + +makeTest ( + +{ + + nodes = + { # Impersonate github.com and api.github.com. + github = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 443 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts = + [ { hostName = "github.com"; + enableSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + } + + { hostName = "api.github.com"; + enableSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/repos/NixOS/nixpkgs/tarball"; + dir = tarball; + } + ]; + extraConfig = + '' + Header set ETag "\"${nixpkgs.rev}\"" + ''; + } + + { hostName = "raw.githubusercontent.com"; + enableSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/NixOS/flake-registry/master"; + dir = registry; + } + ]; + } + ]; + }; + + client = + { config, pkgs, nodes, ... }: + { virtualisation.writableStore = true; + virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; + nix.package = nix; + nix.binaryCaches = [ ]; + environment.systemPackages = [ pkgs.jq ]; + networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = + [ "github.com" "api.github.com" "raw.githubusercontent.com" ]; + security.pki.certificateFiles = [ "${cert}/ca.crt" ]; + }; + }; + + testScript = { nodes }: + '' + use POSIX qw(strftime); + + startAll; + + $github->waitForUnit("httpd.service"); + + $client->succeed("curl -v https://github.com/ >&2"); + + $client->succeed("nix flake list | grep nixpkgs"); + + $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n" + or die "revision mismatch"; + + $client->succeed("nix flake pin nixpkgs"); + + $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2"); + + # Shut down the web server. The flake should be cached on the client. + $github->succeed("systemctl stop httpd.service"); + + my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified"); + strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModified}" or die "time mismatch"; + + $client->succeed("nix build nixpkgs:hello"); + + # The build shouldn't fail even with --tarball-ttl 0 (the server + # being down should not be a fatal error). + $client->succeed("nix build nixpkgs:fuse --tarball-ttl 0"); + ''; + +}) From 06010eaf199005a393f212023ec5e8bc97978537 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Jun 2019 10:34:13 +0200 Subject: [PATCH 0309/2548] Fix fetchTarball with chroot stores Fixes #2405. --- src/libexpr/primops.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 070e72f3a..a9a1a1e35 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2083,12 +2083,12 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, request).path; + auto res = getDownloader()->downloadCached(state.store, request); if (state.allowedPaths) - state.allowedPaths->insert(res); + state.allowedPaths->insert(res.path); - mkString(v, res, PathSet({res})); + mkString(v, res.storePath, PathSet({res.storePath})); } From 415fc233e3f01a3bb3cfac01deb9f4ca4af7fb19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Jun 2019 14:07:25 +0200 Subject: [PATCH 0310/2548] For nixpkgs., use legacyPackages This makes commands like 'nix run nixpkgs.chromium' work again. --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 86e601bc4..1c7debf4e 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -411,7 +411,7 @@ std::vector> SourceExprCommand::parseInstallables( bool static warned; warnOnce(warned, "the syntax 'nixpkgs.' is deprecated; use 'nixpkgs:' instead"); result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), - Strings{"packages." + std::string(s, 8)})); + Strings{"legacyPackages." + std::string(s, 8)})); } else if (auto flakeRef = parseFlakeRef(s, true)) From 9e0f5f803f6cbfe9925cef69a0e58cbf5375bfaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Sat, 8 Jun 2019 00:41:19 +0200 Subject: [PATCH 0311/2548] Daemon: warn when an untrusted user cannot override a setting In a daemon-based Nix setup, some options cannot be overridden by a client unless the client's user is considered trusted. Currently, if an untrusted user tries to override one of those options, we are silently ignoring it. This can be pretty confusing in certain situations. e.g. a user thinks he disabled the sandbox when in reality he did not. We are now sending a warning message letting know the user some options have been ignored. Related to #1761. --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8d63b8f36..973f64a34 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref store, else if (setSubstituters(settings.extraSubstituters)) ; else - debug("ignoring untrusted setting '%s'", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name); } catch (UsageError & e) { warn(e.what()); } From 34fa8ce9179b17cc4cd13ae49b69fccf393271a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Jun 2019 16:34:06 +0200 Subject: [PATCH 0312/2548] nix: Support -j flag --- src/libmain/common-args.cc | 9 +++++++++ src/libmain/shared.cc | 4 ---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..cd752f467 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -175,10 +175,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description("build from source if substitution fails") .set(&(bool&) settings.tryFallback, true); - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); - auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { mkFlag(shortName, longName, description, [=](unsigned int n) { From 5064971ded3e2cc7c6b80d3166270a87a9650c09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 May 2019 21:15:45 +0200 Subject: [PATCH 0313/2548] Fix test failures when $TMPDIR changes (cherry picked from commit c38c726eb5d447c7e9d894d57cd05ac46c173ddd) --- tests/build-dry.sh | 6 +++--- tests/nix-copy-ssh.sh | 2 +- tests/nix-shell.sh | 8 ++++---- tests/placeholders.sh | 2 -- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c..e72533e70 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a4..eb801548d 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399..ee502dddb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc..cd1bb7bc2 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY From 26bc876ae6e7ceff8b37ba5c2c1043ce216cf384 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Jun 2019 16:45:00 +0200 Subject: [PATCH 0314/2548] nix: Add -L alias for --print-build-logs --- src/nix/main.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 4f87ad72b..25e321b86 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -45,6 +45,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("print-build-logs") + .shortName('L') .description("print full build logs on stderr") .set(&printBuildLogs, true); From b693029ca076abf8a4084a053a122ffcdf4d61b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 16 Jun 2019 09:43:20 +0200 Subject: [PATCH 0315/2548] Style fix --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 973f64a34..e88aaf636 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref store, else if (setSubstituters(settings.extraSubstituters)) ; else - warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user.", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } From e84c26564507388f2e8b7c7f2b00eb2b57856da1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 May 2019 22:29:15 +0200 Subject: [PATCH 0316/2548] Run builds in a pseudo-terminal This allows many programs (e.g. gcc, clang, cmake) to print colorized log output (assuming $TERM is set to a value like "xterm"). There are other ways to get colors, in particular setting CLICOLOR_FORCE, but they're less widely supported and can break programs that parse tool output. --- src/libstore/build.cc | 54 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0bd738809..9de7613f8 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -1558,8 +1559,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -2181,7 +2182,43 @@ void DerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -4361,14 +4398,15 @@ void Worker::waitForInput() for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { ssize_t rd = read(k, buffer.data(), buffer.size()); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); From 82ca6ef390b752faf1bf27b22daa34b29a4b00d4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 19 May 2019 16:56:08 +0200 Subject: [PATCH 0317/2548] Set $TERM --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9de7613f8..5a75403cf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2443,6 +2443,9 @@ void DerivationGoal::initEnv() may change that in the future. So tell the builder which file descriptor to use for that. */ env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } From 2743bf0bb1727056ce6d6d6be4e38c3251aac1f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 16 Jun 2019 20:02:40 +0200 Subject: [PATCH 0318/2548] Hopefully fix macOS tests --- src/libstore/build.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5a75403cf..cccf0c7c6 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2190,11 +2190,13 @@ void DerivationGoal::startBuilder() std::string slaveName(ptsname(builderOut.readSide.get())); - if (chmod(slaveName.c_str(), 0600)) - throw SysError("changing mode of pseudoterminal slave"); + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); - if (buildUser && chown(slaveName.c_str(), buildUser->getUID(), 0)) - throw SysError("changing owner of pseudoterminal slave"); + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } #if 0 // Mount the pt in the sandbox so that the "tty" command works. From 3cc1125595d97b4ab7369e37e4ad22f4cfecb8b2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:08:04 +0200 Subject: [PATCH 0319/2548] Another attempt at getting pseudoterminals to work on macOS --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cccf0c7c6..5b38bcf3c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2196,6 +2196,9 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); } #if 0 From 8ea842260b4fd93315d35c5ba94b1ff99ab391d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:43:45 +0200 Subject: [PATCH 0320/2548] Add '--no-net' convenience flag This flag * Disables substituters. * Sets the tarball-ttl to infinity (ensuring e.g. that the flake registry and any downloaded flakes are considered current). * Disables retrying downloads and sets the connection timeout to the minimum. (So it doesn't completely disable downloads at the moment.) --- src/libmain/common-args.cc | 11 +++++++++++ src/libstore/download.cc | 18 +----------------- src/libstore/download.hh | 23 ++++++++++++++++++++++- src/libstore/globals.hh | 2 +- src/libstore/http-binary-cache-store.cc | 1 - 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199..94dd092c5 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "globals.hh" +#include "download.hh" namespace nix { @@ -35,6 +36,16 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([]() { + settings.useSubstitutes = false; + settings.tarballTtl = std::numeric_limits::max(); + downloadSettings.tries = 0; + downloadSettings.connectTimeout = 1; + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0338727c1..5c1705e2f 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -30,23 +30,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 43b1c5c09..c095ad053 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,34 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 80d70fba3..2aecebe3d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,7 +255,7 @@ public: "Secret keys with which to sign local builds."}; Setting tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 105e1dcdd..11c34fdac 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -84,7 +84,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { From 615a9d031d22a6aee64f8511e15685e47b6f8796 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:12:03 +0200 Subject: [PATCH 0321/2548] Add "warning" verbosity level This ensures that "nix" shows warnings. Previously these were hidden because they were at "info" level. --- src/libutil/logging.cc | 3 ++- src/libutil/logging.hh | 1 + src/nix/main.cc | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae..b379306f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102..5f2219445 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/nix/main.cc b/src/nix/main.cc index a1fcb892a..ced87f653 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -102,7 +102,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlError; + verbosity = lvlWarn; settings.verboseBuild = false; evalSettings.pureEval = true; From 04a59769963fe2a28d10ba15de743fe499333c80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:57:22 +0200 Subject: [PATCH 0322/2548] Automatically use --no-net if there are no network interfaces --- src/libmain/common-args.cc | 11 -------- src/nix/main.cc | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 94dd092c5..4c35a4199 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,5 @@ #include "common-args.hh" #include "globals.hh" -#include "download.hh" namespace nix { @@ -36,16 +35,6 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([]() { - settings.useSubstitutes = false; - settings.tarballTtl = std::numeric_limits::max(); - downloadSettings.tries = 0; - downloadSettings.connectTimeout = 1; - }); - std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/nix/main.cc b/src/nix/main.cc index ced87f653..101ed5d5c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,19 +8,52 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include +#include +#include +#include + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) && + !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; + bool useNet = true; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -52,6 +85,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -118,6 +156,23 @@ void mainWrapped(int argc, char * * argv) startProgressBar(args.printBuildLogs); + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + args.command->prepare(); args.command->run(); } From 2467c9837500b26aab5c1dcd3cac12cda44898ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 16:58:59 +0200 Subject: [PATCH 0323/2548] nix app: Search for installable in the 'apps' output I.e. you can write $ nix app blender-bin:blender_2_80 which is equivalent to $ nix app blender-bin:apps.blender_2_80 --- src/nix/command.hh | 12 ++++++++++++ src/nix/installables.cc | 27 +++++++++++++-------------- src/nix/run.cc | 5 +++++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index 659b724c3..3dad64947 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -102,6 +102,18 @@ struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions { return {"defaultPackage"}; } + + virtual Strings getDefaultFlakeAttrPathPrefixes() + { + return { + // As a convenience, look for the attribute in + // 'outputs.packages'. + "packages.", + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + "legacyPackages." + }; + } }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 1c7debf4e..d7dd95606 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -257,14 +257,16 @@ struct InstallableFlake : InstallableValue { FlakeRef flakeRef; Strings attrPaths; - bool searchPackages = false; + Strings prefixes; InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) { } - InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, std::string attrPath) - : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, searchPackages(true) + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, + std::string attrPath, Strings && prefixes) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, + prefixes(prefixes) { } std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } @@ -273,15 +275,8 @@ struct InstallableFlake : InstallableValue { std::vector res; - if (searchPackages) { - // As a convenience, look for the attribute in - // 'outputs.packages'. - res.push_back("packages." + *attrPaths.begin()); - - // As a temporary hack until Nixpkgs is properly converted - // to provide a clean 'packages' set, look in 'legacyPackages'. - res.push_back("legacyPackages." + *attrPaths.begin()); - } + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); for (auto & s : attrPaths) res.push_back(s); @@ -421,7 +416,11 @@ std::vector> SourceExprCommand::parseInstallables( else if ((colon = s.rfind(':')) != std::string::npos) { auto flakeRef = std::string(s, 0, colon); auto attrPath = std::string(s, colon + 1); - result.push_back(std::make_shared(*this, FlakeRef(flakeRef, true), attrPath)); + result.push_back(std::make_shared( + *this, + FlakeRef(flakeRef, true), + attrPath, + getDefaultFlakeAttrPathPrefixes())); } else if (s.find('/') != std::string::npos || s == ".") { @@ -437,7 +436,7 @@ std::vector> SourceExprCommand::parseInstallables( } else - result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), s)); + throw Error("unsupported argument '%s'", s); } } diff --git a/src/nix/run.cc b/src/nix/run.cc index 00a682832..62aae12f6 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -225,6 +225,11 @@ struct CmdApp : InstallableCommand, RunCommon return {"defaultApp"}; } + Strings getDefaultFlakeAttrPathPrefixes() override + { + return {"apps."}; + } + void run(ref store) override { auto state = getEvalState(); From 0d69f7f3f012aceb4c494f3c1cc866b378c5eac1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 17:05:37 +0200 Subject: [PATCH 0324/2548] nix app: Accept arguments Example: $ nix app blender-bin -- --version Blender 2.80 (sub 74) --- src/nix/run.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/nix/run.cc b/src/nix/run.cc index 62aae12f6..d30851d47 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -196,8 +196,11 @@ static RegisterCommand r1(make_ref()); struct CmdApp : InstallableCommand, RunCommon { + std::vector args; + CmdApp() { + expectArgs("args", &args); } std::string name() override @@ -238,7 +241,10 @@ struct CmdApp : InstallableCommand, RunCommon state->realiseContext(app.context); - runProgram(store, app.program, {app.program}); + Strings allArgs{app.program}; + for (auto & i : args) allArgs.push_back(i); + + runProgram(store, app.program, allArgs); } }; From 3b2ebd029cac0b1ce715d8aca44204130c93a869 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 17:31:34 +0200 Subject: [PATCH 0325/2548] nix flake info --json: Revive enumerating the outputs --- src/nix/flake.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index af1a361b3..6f6d3c130 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -234,8 +234,8 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON if (json) { auto json = flakeToJson(flake); -#if 0 auto state = getEvalState(); + auto flake = resolveFlake(); auto vFlake = state->allocValue(); flake::callFlake(*state, flake, *vFlake); @@ -257,7 +257,6 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON }); json["outputs"] = std::move(outputs); -#endif std::cout << json.dump() << std::endl; } else From 9d1207c02c091a00454fcb7266653d18a6023923 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 17:59:57 +0200 Subject: [PATCH 0326/2548] nix flake check: Check apps --- src/libexpr/primops.cc | 14 +++++++------- src/nix/command.hh | 2 ++ src/nix/flake.cc | 29 +++++++++++++++++++++++++++-- src/nix/installables.cc | 23 +++++++++++------------ 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a9a1a1e35..10ce1abf5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -51,21 +51,20 @@ void EvalState::realiseContext(const PathSet & context) PathSet drvs; for (auto & i : context) { - std::pair decoded = decodeContext(i); - Path ctx = decoded.first; + auto [ctx, outputName] = decodeContext(i); assert(store->isStorePath(ctx)); if (!store->isValidPath(ctx)) throw InvalidPathError(ctx); - if (!decoded.second.empty() && nix::isDerivation(ctx)) { - drvs.insert(decoded.first + "!" + decoded.second); + if (!outputName.empty() && nix::isDerivation(ctx)) { + drvs.insert(ctx + "!" + outputName); /* Add the output of this derivation to the allowed paths. */ if (allowedPaths) { - auto drv = store->derivationFromPath(decoded.first); - DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + auto drv = store->derivationFromPath(ctx); + DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + throw Error("derivation '%s' does not have an output named '%s'", ctx, outputName); allowedPaths->insert(i->second.path); } } @@ -80,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context) PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } diff --git a/src/nix/command.hh b/src/nix/command.hh index 3dad64947..0ffbe46f5 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -43,6 +43,8 @@ struct App PathSet context; Path program; // FIXME: add args, sandbox settings, metadata, ... + + App(EvalState & state, Value & vApp); }; struct Installable diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6f6d3c130..6f6d1a0aa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" +#include "derivations.hh" #include #include @@ -301,13 +302,27 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON // FIXME: check meta attributes return drvInfo->queryDrvPath(); } catch (Error & e) { - e.addPrefix(fmt("while checking flake output attribute '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + e.addPrefix(fmt("while checking the derivation '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); throw; } }; PathSet drvPaths; + auto checkApp = [&](const std::string & attrPath, Value & v) { + try { + auto app = App(*state, v); + for (auto & i : app.context) { + auto [drvPath, outputName] = decodeContext(i); + if (!outputName.empty() && nix::isDerivation(drvPath)) + drvPaths.insert(drvPath + "!" + outputName); + } + } catch (Error & e) { + e.addPrefix(fmt("while checking the app definition '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -337,9 +352,19 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON name + "." + (std::string) aCheck.name, *aCheck.value); } + else if (name == "apps") { + state->forceAttrs(vProvide); + for (auto & aCheck : *vProvide.attrs) + checkApp( + name + "." + (std::string) aCheck.name, *aCheck.value); + } + else if (name == "defaultPackage" || name == "devShell") checkDerivation(name, vProvide); + else if (name == "defaultApp") + checkApp(name, vProvide); + } catch (Error & e) { e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); throw; @@ -347,7 +372,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON }); } - if (build) { + if (build && !drvPaths.empty()) { Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); store->buildPaths(drvPaths); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index d7dd95606..feaf57f0c 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -69,26 +69,25 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } -App Installable::toApp(EvalState & state) +App::App(EvalState & state, Value & vApp) { - auto v = toValue(state); + state.forceAttrs(vApp); - state.forceAttrs(*v); - - auto aType = v->attrs->need(state.sType); + auto aType = vApp.attrs->need(state.sType); if (state.forceStringNoCtx(*aType.value, *aType.pos) != "app") throw Error("value does not have type 'app', at %s", *aType.pos); - App app; - - auto aProgram = v->attrs->need(state.symbols.create("program")); - app.program = state.forceString(*aProgram.value, app.context, *aProgram.pos); + auto aProgram = vApp.attrs->need(state.symbols.create("program")); + program = state.forceString(*aProgram.value, context, *aProgram.pos); // FIXME: check that 'program' is in the closure of 'context'. - if (!state.store->isInStore(app.program)) - throw Error("app program '%s' is not in the Nix store", app.program); + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); +} - return app; +App Installable::toApp(EvalState & state) +{ + return App(state, *toValue(state)); } struct InstallableStorePath : Installable From f2fcc163fa6c38566f3a78cd3757ab22f39821fc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 18:05:32 +0200 Subject: [PATCH 0327/2548] nix flake check: Warn about unknown flake outputs --- src/nix/flake.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6f6d1a0aa..8248a5d7b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -365,6 +365,9 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON else if (name == "defaultApp") checkApp(name, vProvide); + else + warn("unknown flake output '%s'", name); + } catch (Error & e) { e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); throw; From 556f33422d0a064ef100fe232f60b42b34bf40e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Jun 2019 09:45:14 +0200 Subject: [PATCH 0328/2548] nix flake check: Ignore legacyPackages --- src/nix/flake.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8248a5d7b..b98cce078 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -365,6 +365,10 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON else if (name == "defaultApp") checkApp(name, vProvide); + else if (name == "legacyPackages") + // FIXME: do getDerivations? + ; + else warn("unknown flake output '%s'", name); From 8a6704d826578d5640d3fca347308298020a6fde Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 26 May 2019 15:59:50 +0200 Subject: [PATCH 0329/2548] Updated documentation --- doc/flakes/design.md | 555 ++++++++++++++++++------------------------- src/nix/flake.cc | 8 - 2 files changed, 235 insertions(+), 328 deletions(-) diff --git a/doc/flakes/design.md b/doc/flakes/design.md index 63198e577..c9520bcbf 100644 --- a/doc/flakes/design.md +++ b/doc/flakes/design.md @@ -2,92 +2,83 @@ ## Goals -* To provide Nix repositories with an easy and standard way to - reference other Nix repositories. +* Standard and easy way for Nix repos to reference other Nix repos as + dependencies -* To allow such references to be queried and updated automatically. +* Discoverability: Be able to query and update these references to Nix repos + automatically -* To provide a replacement for `nix-channel`, `NIX_PATH` and Hydra - jobset definitions. +* To provide a replacement for `nix-channel`, `NIX_PATH` and Hydra jobset + definitions -* To enable reproducible, hermetic evaluation of packages and NixOS - configurations. +* Reproducibility: Evaluate packages and NixOS configurations hermetic by + default -Things that we probably won't do in the initial iteration: +Upcoming but not yet implemented: -* Sophisticated flake versioning, such as the ability to specify - version ranges on dependencies. +* Sophisticated flake versioning, such as the ability to specify version ranges + on dependencies. -* A way to specify the types of values provided by a flake. For the - most part, flakes can provide arbitrary Nix values, but there will - be some standard attribute names (e.g. `packages` must be a set of - installable derivations). +* A way to specify the types of values provided by a flake. For the most part, + flakes can provide arbitrary Nix values, but there will be some standard + attribute names (e.g. `packages` must be a set of installable derivations). ## Overview -* A flake is (usually) a Git repository that contains a file named - `flake.nix` at top-level. +* A flake is (usually) a Git repository that contains a file named `flake.nix` + at top-level -* Flakes *provide* an attribute set of values, such as packages, - Nixpkgs overlays, NixOS modules, library functions, Hydra jobs, - `nix-shell` definitions, etc. +* A flake *provides* an attribute set of values, such as packages, Nixpkgs + overlays, NixOS modules, library functions, Hydra jobs, `nix-shell` + definitions, etc. -* Flakes can *depend* on other flakes. +* Flakes can *depend* on other flakes or other repositories which aren't flakes -* Flakes are referred to using a *flake reference*, which is either a - URL specifying its repository's location - (e.g. `github:NixOS/nixpkgs/release-18.09`) or an identifier - (e.g. `nixpkgs`) looked up in a *lock file* or *flake - registry*. They can also specify revisions, - e.g. `github:NixOS/nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f`. +* Flakes are referred to using a *flake reference*, which is either a URL + specifying its repository's location or an identifier looked up in a *lock + file* or *flake registry*. -* The *flake registry* is a centrally maintained mapping (on - `nixos.org`) from flake identifiers to flake locations - (e.g. `nixpkgs -> github:NixOS/nixpkgs/release-18.09`). +* A *flake registry* is a mapping from flake identifiers to flake locations + (e.g. `nixpkgs -> github:NixOS/nixpkgs/release-18.09`). There is a centrally + maintained flake registry on `nixos.org`. -* A flake can contain a *lock file* (`flake.lock`) used when resolving - the dependencies in `flake.nix`. It maps flake references to - references containing revisions (e.g. `nixpkgs -> +* A flake can contain a *lock file* (`flake.lock`) used when resolving the + dependencies in `flake.nix`. It maps mutable flake references + (e.g. `github:NixOS/nixpkgs/release-18.09`) to references containing revisions + (e.g. `nixpkgs -> github:NixOS/nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f`). -* The `nix` command uses the flake registry as its default - installation source. For example, `nix build nixpkgs.hello` builds the - `hello` package provided by the `nixpkgs` flake listed in the - registry. `nix` will automatically download/upload the registry and - flakes as needed. +* The `nix` command uses the flake registry as its default installation source. + For example, `nix build nixpkgs.hello` builds the `hello` package provided by + the `nixpkgs` flake listed in the registry. `nix` will automatically + download/upload the registry and flakes as needed. * `nix build` without arguments will build the flake in the current directory (or some parent). -* The command `nix flake update` generates/updates `flake.lock` from - `flake.nix`. This should probably also be done automatically when - building from a local flake. +* `nix flake update` generates `flake.lock` from `flake.nix`, ignoring the old + lockfile. -* `nixos-rebuild` will build a configuration from a (locked) - flake. Evaluation will be done in pure mode to ensure there are no - unaccounted inputs. Thus the NixOS configuration can be reproduced - unambiguously from the top-level flake. +* `nixos-rebuild` will build a configuration from a (locked) flake. Evaluation + is done in pure mode to ensure there are no unaccounted inputs. Thus the + NixOS configuration can be reproduced unambiguously from the top-level flake. -* Nix code can query flake metadata such as `commitHash` (the Git - revision) or `date` (the date of the last commit). This is useful - for NixOS to compute the NixOS version string (which will be the - revision of the top-level configuration flake, uniquely identifying - the configuration). +* Nix code can query flake metadata such as `commitHash` (the Git revision) or + `epoch` (the date of the last commit). This is useful for NixOS to compute + the NixOS version string (which will be the revision of the top-level + configuration flake, uniquely identifying the configuration). -* Hydra jobset configurations will consist of a single flake - reference. Thus we can get rid of jobset inputs; any other needed - repositories can be fetched by the top-level flake. The top-level - flake can be locked or unlocked; if some dependencies are unlocked, - then Nix will fetch the latest revision for each. +* Hydra jobset configurations will consist of a single flake reference. Thus we + can get rid of jobset inputs; any other needed repositories can be fetched by + the top-level flake. The top-level flake can be locked or unlocked; if some + dependencies are unlocked, then Nix will fetch the latest revision for each. ## Example flake -A flake is a Git repository that contains a file named -`flake.nix`. For example, here is the `flake.nix` for `dwarffs`, a -small repository that provides a single package and a single NixOS -module. +Let us look at an example of a `flake.nix` file, here for `dwarffs`, a small +repository that provides a single package and a single NixOS module. ```nix { @@ -101,23 +92,26 @@ module. # Some other metadata. description = "A filesystem that fetches DWARF debug info from the Internet on demand"; - # A list of flake references denoting the flakes that this flake - # depends on. Nix will resolve and fetch these flakes and pass them - # as a function argument to `outputs` below. + # The flake dependencies. Nix will resolve and fetch these flakes and pass + # them as a function argument to `outputs` below. # - # `flake:nixpkgs` denotes a flake named `nixpkgs` which is looked up + # "nixpkgs" denotes a flake named `nixpkgs` which is looked up # in the flake registry, or in `flake.lock` inside this flake, if it # exists. inputs = [ flake:nixpkgs ]; + # An attribute set listing dependencies which aren't flakes, also to be passed as + # a function argument to `provides`. + nonFlakeRequires = {}; + # The stuff provided by this flake. Flakes can provide whatever they # want (convention over configuration), but some attributes have - # special meaning to tools / other flakes: for example, `packages` + # special meaning to tools / other flakes. For example, `packages` # is used by the `nix` CLI to search for packages, and # `nixosModules` is used by NixOS to automatically pull in the # modules provided by a flake. # - # `outputs` takes a single argument named `deps` that contains + # `outputs` takes a single argument (`deps`) that contains # the resolved set of flakes. (See below.) outputs = deps: { @@ -153,7 +147,11 @@ module. nixosModules.dwarffs = import ./module.nix deps; # Provide a single Hydra job (`hydraJobs.dwarffs`). - hydraJobs = deps.this.packages; + hydraJobs.build.x86_64-linux = packages.dwarffs; + + # A bunch of things which can be checked (through `nix flake check`) to + # make sure the flake is well-defined. + checks.build = packages.dwarffs; }; } ``` @@ -170,8 +168,11 @@ Similarly, a minimal `flake.nix` for Nixpkgs: outputs = deps: let pkgs = import ./. {}; in + let pkgs = import ./. { system = "x86_64-linux"; }; in { - lib = import ./lib; + lib = (import ./lib) // { + nixosSystem = import ./nixos/lib/eval-config.nix; + }; builders = { inherit (pkgs) stdenv fetchurl; @@ -180,145 +181,124 @@ Similarly, a minimal `flake.nix` for Nixpkgs: packages = { inherit (pkgs) hello nix fuse nlohmann_json boost; }; + + legacyPkgs = pkgs; }; } ``` -Note that `packages` is an unpolluted set of packages: non-package -values like `lib` or `fetchurl` are not part of it. +Note that `packages` is an unpolluted set of packages: non-package values like +`lib` or `fetchurl` are not part of it. +## Flake registries -## Flake identifiers - -A flake has an identifier (e.g. `nixpkgs` or `dwarffs`). +Note: If a flake registry contains an entry `nixpkgs -> github:NixOS/nixpkgs`, +then `nixpkgs/release-18.09` will match to become +`github:NixOS/nixpkgs/release-18.09`. This is referred to as "fuzzymatching". ## Flake references -Flake references are a URI-like syntax to specify the physical -location of a flake (e.g. a Git repository) or to denote a lookup in -the flake registry or lock file. +Flake references are a URI-like syntax to specify the physical location of a +flake (e.g. a Git repository) or to denote a lookup in the flake registry or +lock file. There are four options for the syntax: -* `(flake:)?(/rev-or-ref(/rev)?)?` +* Flake aliases + A flake alias is a name which requires a lookup in a flake + registry or lock file. - Look up a flake by ID in the flake lock file or in the flake - registry. These must specify an actual location for the flake using - the formats listed below. Note that in pure evaluation mode, the - flake registry is empty. + Example: "nixpkgs" - Optionally, the `rev` or `ref` from the dereferenced flake can be - overriden. For example, +* GitHub repositories + A repository which is stored on GitHub can easily be fetched using this type. + Note: + * Only the code in this particular commit is downloaded, not the entire repo + * By default, the commit to download is the last commit on the `master` branch. + See later for how to change this. - > nixpkgs/19.09 + Example: `github:NixOS/nixpkgs` - uses the `19.09` branch of the `nixpkgs` flake's GitHub repository, - while - - > nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - - uses the specified revision. For Git (rather than GitHub) - repositories, both the rev and ref must be given, e.g. - - > nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - -* `github:/(/)?` - - A repository on GitHub. These differ from Git references in that - they're downloaded in a efficient way (via the tarball mechanism) - and that they support downloading a specific revision without - specifying a branch. `rev-or-ref` is either a commit hash (`rev`) - or a branch or tag name (`ref`). The default is `master` if none is - specified. Note that in pure evaluation mode, a commit hash must be - used. - - Flakes fetched in this manner expose `rev` and `date` attributes, - but not `revCount`. - - Examples: - - > github:edolstra/dwarffs - - > github:edolstra/dwarffs/unstable - - > github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 - -* > https:///.git(\?attr(&attr)*)? - - > ssh:///.git(\?attr(&attr)*)? - - > git:///.git(\?attr(&attr)*)? - - > file:///(\?attr(&attr)*)? - - where `attr` is one of `rev=` or `ref=`. - - A Git repository fetched through https. Note that the path must end - in `.git`. The default for `ref` is `master`. - - Examples: - - > https://example.org/my/repo.git - > https://example.org/my/repo.git?ref=release-1.2.3 - > https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - -* > /path.git(\?attr(&attr)*)? - - Like `file://path.git`, but if no `ref` or `rev` is specified, the - (possibly dirty) working tree will be used. Using a working tree is - not allowed in pure evaluation mode. +* `ssh/https/git/file` + These are generic `FlakeRef`s for downloadding git repositories or tarballs. Examples: + - https://example.org/my/repo.git + - ssh://git@github.com:NixOS/nix.git + - git://github.com/edolstra/dwarffs.git + - file:///home/my-user/some-repo/some-repo.git + - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + - file:///.tar.xz - > /path/to/my/repo +* Local, dirty paths + This `FlakeRef` is the equivalent of `file://` used for dirty paths. - > /path/to/my/repo?ref=develop + Example: /path/to/my/repo - > /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - -* > https:///.tar.xz(?hash=) - - > file:///.tar.xz(?hash=) - - A flake distributed as a tarball. In pure evaluation mode, an SRI - hash is mandatory. It exposes a `date` attribute, being the newest - file inside the tarball. - - Example: - - > https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz - - > https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c - -Note: currently, there can be only one flake per Git repository, and -it must be at top-level. In the future, we may want to add a field -(e.g. `dir=`) to specify a subdirectory inside the repository. +Notes: +- Each FlakeRef (except for the Path option) allows for a Git revision (i.e. + commit hash) and/or referenceo(i.e. git branch name) to be added. For + tarbals, an SRI hash needs to be added. + Examples: + * `"nixpkgs/release-18.09"` + * `github:NixOS/nixpkgs/1e9e709953e315ab004951248b186ac8e2306451` + * `git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817` + * file:///.tar.xz(?hash=) +- In full pure mode, no mutable `FlakeRef`s can be used + * No aliases, because they need to be looked up + * `github` requires a specified `rev` + * `ssh/https/git/file` require a specified `ref` _and_ `rev` + * `path` is always mutable +- Flakes don't need to be top-level, but can also reside in a subdirectory. This is shown by adding `dir=` to the `FlakeRef`. + Example: `./foo?dir=bar` ## Flake lock files -This is a JSON file named `flake.lock` that maps flake identifiers -used in the corresponding `flake.nix` to "immutable" flake references; -that is, flake references that contain a revision (for Git -repositories) or a content hash (for tarballs). +A lockfile is a JSON file named `flake.lock` which contains a forrest of +entries mapping `FlakeRef`s to the immutable `FlakeRef` they were resolved to. Example: ```json { - "nixpkgs": "github:NixOS/nixpkgs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553", - "foo": "https://example.org/foo.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c" + "nixpkgs": { + "uri": "github:NixOS/nixpkgs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553", + "content-hash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=" + }, + "foo": { + "uri": "https://example.org/foo.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c", + "content-hash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=" + } } ``` +Lockfiles are used to help resolve the dependencies of a flake. +- `nix build github:<..>` uses the remote lockfile and update it +- `nix build /home/user/dwarffs` uses the local lockfile, updates it and writes the result to file +- `nix flake update ` recreates the lockfile from scratch and writes it to file +- `--no-registries` makes the command pure, also when fetching dependencies +- `--no-save-lock-file`: Several commands will update the lockfile (e.g. `nix + build`). This flag prevents the updated lockfile to be written to file. +- `--recreate-lock-file` makes prevents the current lockfile from being used ## `outputs` -The flake attribute `outputs` is a function that takes an argument -named `deps` and returns a (mostly) arbitrary attrset of values. Some -of the standard result attributes: +The function argument `deps` is an attrset containing all dependencies listed +in `requires` and `nonFlakeRequires` as well as `path` (for the flake's source +code) and an attribute `meta` with: +- `description` +- `commitHash` (not for tarball flakes): The Git commit hash. +- `date`: The timestamp of the most recent commit (for Git repos), or of the + most recently modified file (for tarballs) +- `revCount` (for Git flakes, but not GitHub flakes): The number of ancestors + of the revision. Useful for generating version strings. -* `packages`: A set of installable derivations used by the `nix` - command. That is, commands such as `nix install` ignore all other - flake attributes. +The flake attribute `outputs` is a function that takes an argument named `deps` +and returns an attribute set. Some of the members of this set have protected +names: + +* `packages`: A set of installable derivations used by the `nix` command. That + is, commands such as `nix install` ignore all other flake attributes. It + cannot be a nested set. * `hydraJobs`: Used by Hydra. @@ -329,213 +309,155 @@ of the standard result attributes: we need to avoid a situation where `nixos-rebuild` needs to fetch its own `nixpkgs` just to do `evalModules`.) -* `devShell`: A specification of a development environment in some TBD - format. +* `devShell`: A derivation to create a development environment -The function argument `flakes` is an attrset that contains an -attribute for each dependency specified in `inputs`. (Should it -contain transitive dependencies? Probably not.) Each attribute is an -attrset containing the `outputs` of the dependency, in addition to -the following attributes: - -* `path`: The path to the flake's source code. Useful when you want to - use non-Nix artifacts from the flake, or if you want to *store* the - source code of the dependency in a derivation. (For example, we - could store the sources of all flake dependencies in a NixOS system - configuration, as a generalization of - `system.copySystemConfiguration`.) - -* `meta`: An attrset containing the following: - - * `description` - - * `commitHash` (or `rev`?) (not for tarball flakes): The Git commit - hash. - - * `date`: The timestamp of the most recent commit (for Git - repositories), or the timestamp of the most recently modified file - (for tarballs). - - * `revCount` (for Git flakes, but not GitHub flakes): The number of - ancestors of the revision. Useful for generating version strings. - - -## Non-flake dependencies - -It may be useful to pull in repositories that are not flakes -(i.e. don't contain a `flake.nix`). This could be done in two ways: - -* Allow flakes not to have a `flake.nix` file, in which case it's a - flake with no inputs and no outputs. The downside of this - approach is that we can't detect accidental use of a non-flake - repository. (Also, we need to conjure up an identifier somehow.) - -* Add a flake attribute to specifiy non-flake dependencies, e.g. - - > nonFlakeInputs.foobar = github:foo/bar; +* `self`: The result of the flake's output which is passed to itself + Example: `self.outputs.foo` works. ## Flake registry -The flake registry maps flake IDs to flake references (where the -latter cannot be another indirection, i.e. it must not be a -`flake:` reference). - -The default registry is kept at -`https://nixos.org/flake-registry.json`. It looks like this: +A flake registry is a JSON file mapping flake references to flake references. +The default/global registry is kept at +`https://github.com/NixOS/flake-registry/blob/master/flake-registry.json` and +looks like this: ```json { - "version": 1, "flakes": { "dwarffs": { "uri": "github:edolstra/dwarffs/flake" }, + "nix": { + "uri": "github:NixOS/nix/flakes" + }, "nixpkgs": { - "uri": "github:NixOS/nixpkgs/release-18.09" + "uri": "github:edolstra/nixpkgs/release-19.03" + }, + "hydra": { + "uri": "github:NixOS/hydra/flake" + }, + "patchelf": { + "uri": "github:NixOS/patchelf" } - } + }, + "version": 1 } ``` -Nix automatically (re)downloads the registry. The downloaded file is a -GC root so the registry remains available if nixos.org is unreachable. -TBD: when to redownload? +Nix automatically (re)downloads this file whenever you have network access. The +downloaded file is a GC root so the registry remains available if nixos.org is +unreachable. + +In addition to a global registry, there is also a user registry stored in +`~/.config/nix/registry.json`. ## Nix UI -Commands for registry / user flake configuration: +There is a list of new commands added to the `nix` CLI: -* `nix flake list`: Show all flakes in the registry. +* `nix flake list`: Show all flakes in the registry -* `nix flake add `: Add or override a flake to/in the - user's flake configuration (`~/.config/nix/flakes.nix`). For - example, `nix flake add nixpkgs/nixos-18.03` overrides the `nixpkgs` - flake to use the `nixos-18.03` branch. There should also be a way to - add multiple branches/revisions of the same flake by giving them a - different ID, e.g. `nix flake add --id nixpkgs-ancient - nixpkgs/nixos-16.03`). +* `nix flake add `: Add or override a flake + to/in the user flake registry. -* `nix flake remove `: Remove a flake from the user's flake - configuration. Any flake with the same ID in the registry remains - available. +* `nix flake remove `: Remove a FlakeRef from the user flake + registry. -* `nix flake lock `: Lock a flake. For example, `nix flake - lock nixpkgs` pins `nixpkgs` to the current revision. +* `nix flake pin `: Look up to which immutable FlakeRef the + alias FlakeRef maps to currently, and store that map in the user registry. + Example: `nix flake pin github:NixOS/nixpkgs` will create an entry + `github:NixOS/nixpkgs -> + github:NixOS/nixpkgs/444f22ca892a873f76acd88d5d55bdc24ed08757`. -Commands for creating/modifying a flake: +* `nix flake init`: Create a `flake.nix` in the current directory -* `nix flake init`: Create a `flake.nix` in the current directory. - -* `nix flake update`: Update the lock file for the `flake.nix` in the - current directory. In most cases, this should be done - automatically. (E.g. `nix build` should automatically update the - lock file is a new dependency is added to `flake.nix`.) +* `nix flake update`: Recreate the lock file from scratch, from the `flake.nix`. * `nix flake check`: Do some checks on the flake, e.g. check that all `packages` are really packages. -* `nix flake clone`: Do a Git clone of the flake repository. This is a - convenience to easily start hacking on a flake. E.g. `nix flake - clone dwarffs` clones the `dwarffs` GitHub repository to `./dwarffs`. - -TODO: maybe the first set of commands should have a different name -from the second set. +* `nix flake clone`: `git clone` the flake repo Flags / configuration options: -* `--flakes (=)*`: add/override some flakes. +* `--flakes (=)*`: add/override some + FlakeRef -* (In `nix`) `--flake `: set the specified flake as the - installation source. E.g. `nix build --flake ./my-nixpkgs hello`. +* `--flake `: set the specified flake as the installation source + E.g. `nix build --flake ./my-nixpkgs hello`. -The default installation source in `nix` is the `packages` from all -flakes in the registry, that is: +The default installation source in `nix` is the `packages` from all flakes in +the registry, that is: ``` builtins.mapAttrs (flakeName: flakeInfo: (getFlake flakeInfo.uri).${flakeName}.outputs.packages or {}) builtins.flakeRegistry ``` -(where `builtins.flakeRegistry` is the global registry with user -overrides applied, and `builtins.getFlake` downloads a flake and -resolves its dependencies.) - -It may be nice to extend the default installation source with the -`packages` from the flake in the current directory, so that - -> nix build hello - -does something similar to the old - -> nix-build -A hello - -Specifically, it builds `packages.hello` from the flake in the current -directory. Of course, this creates some ambiguity if there is a flake -in the registry named `hello`. - -Maybe the command - -> nix dev-shell - -should do something like use `outputs.devShell` to initialize the -shell, but probably we should ditch `nix shell` / `nix-shell` for -direnv. +where `builtins.flakeRegistry` is the global registry with user overrides +applied, and `builtins.getFlake` downloads a flake and resolves its +dependencies. ## Pure evaluation and caching -Flake evaluation should be done in pure mode. Thus: +Flake evaluation is done in pure mode. Thus: -* Flakes cannot do `NIX_PATH` lookups via the `<...>` syntax. +* Flakes cannot use `NIX_PATH` via the `<...>` syntax. -* They can't read random stuff from non-flake directories, such as +* Flakes cannot read random stuff from non-flake directories, such as `~/.nix/config.nix` or overlays. -This enables aggressive caching or precomputation of Nixpkgs package -sets. For example, for a particular Nixpkgs flake closure (as -identified by, say, a hash of the fully-qualified flake references -after dependency resolution) and system type, an attribute like -`packages.hello` should always evaluate to the same derivation. So we -can: +This enables aggressive caching or precomputation of Nixpkgs package sets. For +example, for a particular Nixpkgs flake closure (as identified by, say, a hash +of the fully-qualified flake references after dependency resolution) and system +type, an attribute like `packages.hello` should always evaluate to the same +derivation. So we can: -* Keep a local evaluation cache (say `~/.cache/nix/eval.sqlite`) +* Keep a local evaluation cache (say `~/.cache/nix/eval-cache-v1.sqlite`) mapping `() -> (, , )`. -* Download a precomputed cache - (e.g. `https://releases.nixos.org/eval/.sqlite`). So - a command like `nix search` could avoid evaluating Nixpkgs entirely. +* Download a precomputed cache, e.g. + `https://releases.nixos.org/eval/.sqlite`. So a command + like `nix search` could avoid evaluating Nixpkgs entirely. -Of course, this doesn't allow overlays. With pure evaluation, the only -way to have these is to define a top-level flake that depends on the -Nixpkgs flake and somehow passes in a set of overlays. - -TODO: in pure mode we have to pass the system type explicitly! +Of course, this doesn't allow overlays. With pure evaluation, the only way to +have these is to define a top-level flake that depends on the Nixpkgs flake and +somehow passes in a set of overlays. ## Hydra jobset dependencies -Hydra can use the flake dependency resolution mechanism to fetch -dependencies. This allows us to get rid of jobset configuration in the -web interface: a jobset only requires a flake reference. That is, *a -jobset is a flake*. Hydra then just builds the `hydraJobs` attrset -`provide`d by the flake. (It omitted, maybe it can build `packages`.) +Hydra can use the flake dependency resolution mechanism to fetch dependencies. +This allows us to get rid of jobset configuration in the web interface: a +jobset only requires a flake reference. That is, a jobset *is* a flake. Hydra +then just builds the `hydraJobs` attrset ## NixOS system configuration -NixOS currently contains a lot of modules that really should be moved -into their own repositories. For example, it contains a Hydra module -that duplicates the one in the Hydra repository. Also, we want -reproducible evaluation for NixOS system configurations. So NixOS -system configurations should be stored as flakes in (local) Git -repositories. +NixOS currently contains a lot of modules that really should be moved into +their own repositories. For example, it contains a Hydra module that duplicates +the one in the Hydra repository. Also, we want reproducible evaluation for +NixOS system configurations. So NixOS system configurations should be stored as +flakes in (local) Git repositories. `my-system/flake.nix`: - ```nix { + name = "my-system"; + + epoch = 201906; + + inputs = + [ "nixpkgs/nixos-18.09" + "dwarffs" + "hydra" + ... lots of other module flakes ... + ]; + outputs = flakes: { nixosSystems.default = flakes.nixpkgs.lib.evalModules { @@ -549,13 +471,6 @@ repositories. ]; }; }; - - inputs = - [ "nixpkgs/nixos-18.09" - "dwarffs" - "hydra" - ... lots of other module flakes ... - ]; } ``` @@ -563,5 +478,5 @@ We can then build the system: ``` nixos-rebuild switch --flake ~/my-system ``` -This performs dependency resolution starting at `~/my-system/flake.nix` -and builds the `system` attribute in `nixosSystems.default`. +This performs dependency resolution starting at `~/my-system/flake.nix` and +builds the `system` attribute in `nixosSystems.default`. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8d6716391..b673ca73e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -128,14 +128,6 @@ static void printNonFlakeInfo(const NonFlake & nonFlake) printSourceInfo(nonFlake.sourceInfo); } -static nlohmann::json nonFlakeToJson(const NonFlake & nonFlake) -{ - nlohmann::json j; - j["id"] = nonFlake.alias; - sourceInfoToJson(nonFlake.sourceInfo, j); - return j; -} - // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { From a0de58f471c9087d8e6cc60a6078f9940a125b15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Jun 2019 16:01:35 +0200 Subject: [PATCH 0330/2548] Make subcommand construction in MultiCommand lazy --- src/libutil/args.cc | 15 ++++---- src/libutil/args.hh | 11 ++++-- src/nix/add-to-store.cc | 7 +--- src/nix/build.cc | 7 +--- src/nix/cat.cc | 14 +------ src/nix/command.cc | 2 +- src/nix/command.hh | 15 ++++++-- src/nix/copy.cc | 7 +--- src/nix/doctor.cc | 7 +--- src/nix/dump-path.cc | 7 +--- src/nix/edit.cc | 7 +--- src/nix/eval.cc | 7 +--- src/nix/flake.cc | 79 ++++++-------------------------------- src/nix/hash.cc | 26 +++---------- src/nix/log.cc | 11 +----- src/nix/ls.cc | 14 +------ src/nix/optimise-store.cc | 11 +----- src/nix/path-info.cc | 7 +--- src/nix/ping-store.cc | 7 +--- src/nix/repl.cc | 4 +- src/nix/run.cc | 14 +------ src/nix/search.cc | 7 +--- src/nix/shell.cc | 16 +------- src/nix/show-config.cc | 11 +----- src/nix/show-derivation.cc | 7 +--- src/nix/sigs.cc | 14 +------ src/nix/upgrade-nix.cc | 7 +--- src/nix/verify.cc | 7 +--- src/nix/why-depends.cc | 7 +--- 29 files changed, 73 insertions(+), 282 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 2837dacc9..217495c26 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -215,17 +215,15 @@ void Command::printHelp(const string & programName, std::ostream & out) } } -MultiCommand::MultiCommand(const std::vector> & _commands) +MultiCommand::MultiCommand(const Commands & commands) + : commands(commands) { - for (auto & command : _commands) - commands.emplace(command->name(), command); - expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { assert(!command); auto i = commands.find(ss[0]); if (i == commands.end()) throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second; + command = i->second(); }}); } @@ -246,10 +244,11 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out) out << "Available commands:\n"; Table2 table; - for (auto & command : commands) { - auto descr = command.second->description(); + for (auto & i : commands) { + auto command = i.second(); + auto descr = command->description(); if (!descr.empty()) - table.push_back(std::make_pair(command.second->name(), descr)); + table.push_back(std::make_pair(command->name(), descr)); } printTable(out, table); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index bf69bf4b6..8497eaf71 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -192,7 +192,12 @@ public: run() method. */ struct Command : virtual Args { - virtual std::string name() = 0; +private: + std::string _name; + +public: + std::string name() { return _name; } + virtual void prepare() { }; virtual void run() = 0; @@ -209,7 +214,7 @@ struct Command : virtual Args void printHelp(const string & programName, std::ostream & out) override; }; -typedef std::map> Commands; +typedef std::map()>> Commands; /* An argument parser that supports multiple subcommands, i.e. ‘ ’. */ @@ -220,7 +225,7 @@ public: std::shared_ptr command; - MultiCommand(const std::vector> & commands); + MultiCommand(const Commands & commands); void printHelp(const string & programName, std::ostream & out) override; diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index e86b96e3f..296b2c7e4 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -22,11 +22,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand .dest(&namePart); } - std::string name() override - { - return "add-to-store"; - } - std::string description() override { return "add a path to the Nix store"; @@ -58,4 +53,4 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("add-to-store"); diff --git a/src/nix/build.cc b/src/nix/build.cc index c08ec0e62..d8ce8cc80 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -25,11 +25,6 @@ struct CmdBuild : MixDryRun, InstallablesCommand .set(&outLink, Path("")); } - std::string name() override - { - return "build"; - } - std::string description() override { return "build a derivation or fetch a store path"; @@ -72,4 +67,4 @@ struct CmdBuild : MixDryRun, InstallablesCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("build"); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index a35f640d8..851f90abd 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -28,11 +28,6 @@ struct CmdCatStore : StoreCommand, MixCat expectArg("path", &path); } - std::string name() override - { - return "cat-store"; - } - std::string description() override { return "print the contents of a store file on stdout"; @@ -54,11 +49,6 @@ struct CmdCatNar : StoreCommand, MixCat expectArg("path", &path); } - std::string name() override - { - return "cat-nar"; - } - std::string description() override { return "print the contents of a file inside a NAR file"; @@ -70,5 +60,5 @@ struct CmdCatNar : StoreCommand, MixCat } }; -static RegisterCommand r1(make_ref()); -static RegisterCommand r2(make_ref()); +static auto r1 = registerCommand("cat-store"); +static auto r2 = registerCommand("cat-nar"); diff --git a/src/nix/command.cc b/src/nix/command.cc index 5967ab36c..89fa0cba4 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -4,7 +4,7 @@ namespace nix { -std::vector> * RegisterCommand::commands = 0; +Commands * RegisterCommand::commands = nullptr; StoreCommand::StoreCommand() { diff --git a/src/nix/command.hh b/src/nix/command.hh index 0ffbe46f5..59c6f8578 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -190,15 +190,22 @@ struct StorePathCommand : public InstallablesCommand /* A helper class for registering commands globally. */ struct RegisterCommand { - static std::vector> * commands; + static Commands * commands; - RegisterCommand(ref command) + RegisterCommand(const std::string & name, + std::function()> command) { - if (!commands) commands = new std::vector>; - commands->push_back(command); + if (!commands) commands = new Commands; + commands->emplace(name, command); } }; +template +static RegisterCommand registerCommand(const std::string & name) +{ + return RegisterCommand(name, [](){ return make_ref(); }); +} + Buildables build(ref store, RealiseMode mode, std::vector> installables); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 96bd453d8..5f9051f40 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -42,11 +42,6 @@ struct CmdCopy : StorePathsCommand .set(&substitute, Substitute); } - std::string name() override - { - return "copy"; - } - std::string description() override { return "copy paths between Nix stores"; @@ -97,4 +92,4 @@ struct CmdCopy : StorePathsCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("copy"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7b5444619..f2cf04758 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -20,11 +20,6 @@ struct CmdDoctor : StoreCommand { bool success = true; - std::string name() override - { - return "doctor"; - } - std::string description() override { return "check your system for potential problems"; @@ -121,4 +116,4 @@ struct CmdDoctor : StoreCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("doctore"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index f411c0cb7..90f1552d9 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -5,11 +5,6 @@ using namespace nix; struct CmdDumpPath : StorePathCommand { - std::string name() override - { - return "dump-path"; - } - std::string description() override { return "dump a store path to stdout (in NAR format)"; @@ -33,4 +28,4 @@ struct CmdDumpPath : StorePathCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("dump-path"); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d..c62b35c46 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -10,11 +10,6 @@ using namespace nix; struct CmdEdit : InstallableCommand { - std::string name() override - { - return "edit"; - } - std::string description() override { return "open the Nix expression of a Nix package in $EDITOR"; @@ -78,4 +73,4 @@ struct CmdEdit : InstallableCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("edit"); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index b7058361c..524bac304 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,11 +18,6 @@ struct CmdEval : MixJSON, InstallableCommand mkFlag(0, "raw", "print strings unquoted", &raw); } - std::string name() override - { - return "eval"; - } - std::string description() override { return "evaluate a Nix expression"; @@ -74,4 +69,4 @@ struct CmdEval : MixJSON, InstallableCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("eval"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6fc56827f..91c6b4276 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -49,11 +49,6 @@ public: struct CmdFlakeList : EvalCommand { - std::string name() override - { - return "list"; - } - std::string description() override { return "list available Nix flakes"; @@ -133,11 +128,6 @@ static void printNonFlakeInfo(const NonFlake & nonFlake) // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { - std::string name() override - { - return "deps"; - } - std::string description() override { return "list informaton about dependencies"; @@ -171,11 +161,6 @@ struct CmdFlakeDeps : FlakeCommand struct CmdFlakeUpdate : FlakeCommand { - std::string name() override - { - return "update"; - } - std::string description() override { return "update flake lock file"; @@ -209,11 +194,6 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, struct CmdFlakeInfo : FlakeCommand, MixJSON { - std::string name() override - { - return "info"; - } - std::string description() override { return "list info about a given flake"; @@ -269,11 +249,6 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON .set(&build, false); } - std::string name() override - { - return "check"; - } - std::string description() override { return "check whether the flake evaluates and run its tests"; @@ -383,11 +358,6 @@ struct CmdFlakeAdd : MixEvalArgs, Command FlakeUri alias; FlakeUri uri; - std::string name() override - { - return "add"; - } - std::string description() override { return "upsert flake in user flake registry"; @@ -414,11 +384,6 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command { FlakeUri alias; - std::string name() override - { - return "remove"; - } - std::string description() override { return "remove flake from user flake registry"; @@ -442,11 +407,6 @@ struct CmdFlakePin : virtual Args, EvalCommand { FlakeUri alias; - std::string name() override - { - return "pin"; - } - std::string description() override { return "pin flake require in user flake registry"; @@ -482,11 +442,6 @@ struct CmdFlakePin : virtual Args, EvalCommand struct CmdFlakeInit : virtual Args, Command { - std::string name() override - { - return "init"; - } - std::string description() override { return "create a skeleton 'flake.nix' file in the current directory"; @@ -514,11 +469,6 @@ struct CmdFlakeClone : FlakeCommand { Path destDir; - std::string name() override - { - return "clone"; - } - std::string description() override { return "clone flake repository"; @@ -541,25 +491,20 @@ struct CmdFlakeClone : FlakeCommand struct CmdFlake : virtual MultiCommand, virtual Command { CmdFlake() - : MultiCommand({make_ref() - , make_ref() - , make_ref() - , make_ref() - //, make_ref() - , make_ref() - , make_ref() - , make_ref() - , make_ref() - , make_ref() - }) + : MultiCommand({ + {"list", []() { return make_ref(); }}, + {"update", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, + {"check", []() { return make_ref(); }}, + {"add", []() { return make_ref(); }}, + {"remove", []() { return make_ref(); }}, + {"pin", []() { return make_ref(); }}, + {"init", []() { return make_ref(); }}, + {"clone", []() { return make_ref(); }}, + }) { } - std::string name() override - { - return "flake"; - } - std::string description() override { return "manage Nix flakes"; @@ -578,4 +523,4 @@ struct CmdFlake : virtual MultiCommand, virtual Command } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("flake"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index af4105e28..1b3ba729e 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -26,11 +26,6 @@ struct CmdHash : Command expectArgs("paths", &paths); } - std::string name() override - { - return mode == mFile ? "hash-file" : "hash-path"; - } - std::string description() override { return mode == mFile @@ -49,8 +44,8 @@ struct CmdHash : Command } }; -static RegisterCommand r1(make_ref(CmdHash::mFile)); -static RegisterCommand r2(make_ref(CmdHash::mPath)); +static RegisterCommand r1("hash-file", [](){ return make_ref(CmdHash::mFile); }); +static RegisterCommand r2("hash-path", [](){ return make_ref(CmdHash::mPath); }); struct CmdToBase : Command { @@ -66,15 +61,6 @@ struct CmdToBase : Command expectArgs("strings", &args); } - std::string name() override - { - return - base == Base16 ? "to-base16" : - base == Base32 ? "to-base32" : - base == Base64 ? "to-base64" : - "to-sri"; - } - std::string description() override { return fmt("convert a hash to %s representation", @@ -91,10 +77,10 @@ struct CmdToBase : Command } }; -static RegisterCommand r3(make_ref(Base16)); -static RegisterCommand r4(make_ref(Base32)); -static RegisterCommand r5(make_ref(Base64)); -static RegisterCommand r6(make_ref(SRI)); +static RegisterCommand r3("to-base16", [](){ return make_ref(Base16); }); +static RegisterCommand r4("to-base32", [](){ return make_ref(Base32); }); +static RegisterCommand r5("to-base64", [](){ return make_ref(Base64); }); +static RegisterCommand r6("to-sri", [](){ return make_ref(SRI); }); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) diff --git a/src/nix/log.cc b/src/nix/log.cc index f07ec4e93..122a3d690 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -8,15 +8,6 @@ using namespace nix; struct CmdLog : InstallableCommand { - CmdLog() - { - } - - std::string name() override - { - return "log"; - } - std::string description() override { return "show the build log of the specified packages or paths, if available"; @@ -68,4 +59,4 @@ struct CmdLog : InstallableCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("log"); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d089be42f..9408cc9da 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -100,11 +100,6 @@ struct CmdLsStore : StoreCommand, MixLs }; } - std::string name() override - { - return "ls-store"; - } - std::string description() override { return "show information about a store path"; @@ -136,11 +131,6 @@ struct CmdLsNar : Command, MixLs }; } - std::string name() override - { - return "ls-nar"; - } - std::string description() override { return "show information about the contents of a NAR file"; @@ -152,5 +142,5 @@ struct CmdLsNar : Command, MixLs } }; -static RegisterCommand r1(make_ref()); -static RegisterCommand r2(make_ref()); +static auto r1 = registerCommand("ls-store"); +static auto r2 = registerCommand("ls-nar"); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index 725fb75a1..fed012b04 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -8,15 +8,6 @@ using namespace nix; struct CmdOptimiseStore : StoreCommand { - CmdOptimiseStore() - { - } - - std::string name() override - { - return "optimise-store"; - } - std::string description() override { return "replace identical files in the store by hard links"; @@ -38,4 +29,4 @@ struct CmdOptimiseStore : StoreCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("optimise-store"); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index dea5f0557..2cb718f12 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -24,11 +24,6 @@ struct CmdPathInfo : StorePathsCommand, MixJSON mkFlag(0, "sigs", "show signatures", &showSigs); } - std::string name() override - { - return "path-info"; - } - std::string description() override { return "query information about store paths"; @@ -130,4 +125,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("path-info"); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 310942574..3a2e542a3 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -6,11 +6,6 @@ using namespace nix; struct CmdPingStore : StoreCommand { - std::string name() override - { - return "ping-store"; - } - std::string description() override { return "test whether a store can be opened"; @@ -32,4 +27,4 @@ struct CmdPingStore : StoreCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("ping-store"); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d8f812149..ce17a2b9f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -764,8 +764,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs expectArgs("files", &files); } - std::string name() override { return "repl"; } - std::string description() override { return "start an interactive environment for evaluating Nix expressions"; @@ -779,6 +777,6 @@ struct CmdRepl : StoreCommand, MixEvalArgs } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("repl"); } diff --git a/src/nix/run.cc b/src/nix/run.cc index d30851d47..9c15b6749 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -99,11 +99,6 @@ struct CmdRun : InstallablesCommand, RunCommon .handler([&](std::vector ss) { unset.insert(ss.front()); }); } - std::string name() override - { - return "run"; - } - std::string description() override { return "run a shell in which the specified packages are available"; @@ -192,7 +187,7 @@ struct CmdRun : InstallablesCommand, RunCommon } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("run"); struct CmdApp : InstallableCommand, RunCommon { @@ -203,11 +198,6 @@ struct CmdApp : InstallableCommand, RunCommon expectArgs("args", &args); } - std::string name() override - { - return "app"; - } - std::string description() override { return "run a Nix application"; @@ -248,7 +238,7 @@ struct CmdApp : InstallableCommand, RunCommon } }; -static RegisterCommand r2(make_ref()); +static auto r2 = registerCommand("app"); void chrootHelper(int argc, char * * argv) { diff --git a/src/nix/search.cc b/src/nix/search.cc index 55f8d106a..70de717d1 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -52,11 +52,6 @@ struct CmdSearch : SourceExprCommand, MixJSON .handler([&]() { writeCache = false; useCache = false; }); } - std::string name() override - { - return "search"; - } - std::string description() override { return "query available packages"; @@ -282,4 +277,4 @@ struct CmdSearch : SourceExprCommand, MixJSON } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("search"); diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 2ccad930f..f42947b7c 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -177,12 +177,6 @@ struct Common : InstallableCommand struct CmdDevShell : Common { - - std::string name() override - { - return "dev-shell"; - } - std::string description() override { return "run a bash shell that provides the build environment of a derivation"; @@ -240,12 +234,6 @@ struct CmdDevShell : Common struct CmdPrintDevEnv : Common { - - std::string name() override - { - return "print-dev-env"; - } - std::string description() override { return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; @@ -279,5 +267,5 @@ struct CmdPrintDevEnv : Common } }; -static RegisterCommand r1(make_ref()); -static RegisterCommand r2(make_ref()); +static auto r1 = registerCommand("print-dev-env"); +static auto r2 = registerCommand("dev-shell"); diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 86638b50d..87544f937 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -8,15 +8,6 @@ using namespace nix; struct CmdShowConfig : Command, MixJSON { - CmdShowConfig() - { - } - - std::string name() override - { - return "show-config"; - } - std::string description() override { return "show the Nix configuration"; @@ -37,4 +28,4 @@ struct CmdShowConfig : Command, MixJSON } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index ee94fded3..6065adc4d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -22,11 +22,6 @@ struct CmdShowDerivation : InstallablesCommand .set(&recursive, true); } - std::string name() override - { - return "show-derivation"; - } - std::string description() override { return "show the contents of a store derivation"; @@ -116,4 +111,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("show-derivation"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index b1825c412..23bc83ad0 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -22,11 +22,6 @@ struct CmdCopySigs : StorePathsCommand .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); } - std::string name() override - { - return "copy-sigs"; - } - std::string description() override { return "copy path signatures from substituters (like binary caches)"; @@ -93,7 +88,7 @@ struct CmdCopySigs : StorePathsCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("copy-sigs"); struct CmdSignPaths : StorePathsCommand { @@ -109,11 +104,6 @@ struct CmdSignPaths : StorePathsCommand .dest(&secretKeyFile); } - std::string name() override - { - return "sign-paths"; - } - std::string description() override { return "sign the specified paths"; @@ -146,4 +136,4 @@ struct CmdSignPaths : StorePathsCommand } }; -static RegisterCommand r3(make_ref()); +static auto r2 = registerCommand("sign-paths"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 35c44a70c..13d8504a6 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -30,11 +30,6 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .dest(&storePathsUrl); } - std::string name() override - { - return "upgrade-nix"; - } - std::string description() override { return "upgrade Nix to the latest stable version"; @@ -157,4 +152,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("upgrade-nix"); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 7ef571561..f55766eda 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -29,11 +29,6 @@ struct CmdVerify : StorePathsCommand mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } - std::string name() override - { - return "verify"; - } - std::string description() override { return "verify the integrity of store paths"; @@ -175,4 +170,4 @@ struct CmdVerify : StorePathsCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("verify"); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 32ba5a1ad..3d13a77e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -44,11 +44,6 @@ struct CmdWhyDepends : SourceExprCommand .set(&all, true); } - std::string name() override - { - return "why-depends"; - } - std::string description() override { return "show why a package has another package in its closure"; @@ -264,4 +259,4 @@ struct CmdWhyDepends : SourceExprCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("why-depends"); From 59714a15e0b36c93a7c19b71344a9fccf7b9840d Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Tue, 11 Jun 2019 13:09:06 +0200 Subject: [PATCH 0331/2548] FuzzyMatching works Fixes #2843 --- src/libexpr/flake/flake.cc | 1 + tests/flakes.sh | 56 +++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 0018a0d07..30ab9a876 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -104,6 +104,7 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const auto newRef = j->second; newRef.ref = flakeRef.ref; newRef.rev = flakeRef.rev; + newRef.subdir = flakeRef.subdir; return updateFlakeRef(state, newRef, registries, pastSearches); } } diff --git a/tests/flakes.sh b/tests/flakes.sh index c380b405b..85ba7c8c9 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -15,6 +15,7 @@ registry=$TEST_ROOT/registry.json flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 flake3Dir=$TEST_ROOT/flake3 +flake4Dir=$TEST_ROOT/flake4 nonFlakeDir=$TEST_ROOT/nonFlake for repo in $flake1Dir $flake2Dir $flake3Dir $nonFlakeDir; do @@ -101,6 +102,12 @@ cat > $registry < $registry < $flake3Dir/flake.nix < \$out + ''; + }; + }; +} +EOF +git -C $flake3Dir add flake.nix +git -C $flake3Dir commit -m 'Remove packages.xyzzy' +git -C $flake3Dir checkout master + +# Test whether fuzzy-matching works for IsAlias +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:xyzzy) + +# Test whether fuzzy-matching works for IsGit +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:xyzzy) +nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:sth From e3552f2bcf246ea8f0dae0c076f1f67e86953746 Mon Sep 17 00:00:00 2001 From: Nick Van den Broeck Date: Wed, 19 Jun 2019 14:48:40 +0200 Subject: [PATCH 0332/2548] Added tests for the `nix flake` CLI --- tests/flakes.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 85ba7c8c9..0571c2a62 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -16,9 +16,10 @@ flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 flake3Dir=$TEST_ROOT/flake3 flake4Dir=$TEST_ROOT/flake4 +flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake -for repo in $flake1Dir $flake2Dir $flake3Dir $nonFlakeDir; do +for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do rm -rf $repo $repo.tmp mkdir $repo git -C $repo init @@ -76,6 +77,10 @@ cat > $flake3Dir/flake.nix < Date: Wed, 19 Jun 2019 23:37:40 +0200 Subject: [PATCH 0333/2548] Initialize Command::_name --- src/libutil/args.cc | 2 ++ src/libutil/args.hh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 217495c26..ba15ea571 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -224,6 +224,7 @@ MultiCommand::MultiCommand(const Commands & commands) if (i == commands.end()) throw UsageError("'%s' is not a recognised command", ss[0]); command = i->second(); + command->_name = ss[0]; }}); } @@ -246,6 +247,7 @@ void MultiCommand::printHelp(const string & programName, std::ostream & out) Table2 table; for (auto & i : commands) { auto command = i.second(); + command->_name = i.first; auto descr = command->description(); if (!descr.empty()) table.push_back(std::make_pair(command->name(), descr)); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 8497eaf71..a083c4ce8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -195,7 +195,10 @@ struct Command : virtual Args private: std::string _name; + friend class MultiCommand; + public: + std::string name() { return _name; } virtual void prepare() { }; From 74a65d313f5c9f4d504ea9e2caadfe0ff83776d5 Mon Sep 17 00:00:00 2001 From: Bruno Bieth Date: Thu, 20 Jun 2019 14:47:45 +0200 Subject: [PATCH 0334/2548] Nix uses the CPP SDK, not Java --- doc/manual/packages/s3-substituter.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index e7589ffdb..1722090ef 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -113,7 +113,7 @@ the S3 URL: exactly s3://example-nix-cache. Nix will use the default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain for authenticating requests to Amazon S3. @@ -138,7 +138,7 @@ the S3 URL: be s3://example-nix-cache. Nix will use the default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain for authenticating requests to Amazon S3. From 29ccb2e9697ee2184012dd13854e487928ae4441 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 12:49:18 +0200 Subject: [PATCH 0335/2548] Fix 32-bit overflow with --no-net --no-net causes tarballTtl to be set to the largest 32-bit integer, which causes comparison like 'time + tarballTtl < other_time' to fail on 32-bit systems. So cast them to 64-bit first. https://hydra.nixos.org/build/95076624 --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 13 +++++-------- src/libstore/download.cc | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 10f6b6f72..f0f5b2a51 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -135,7 +135,7 @@ GitInfo exportGit(ref store, std::string uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 596047ce3..c791443c3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -77,7 +77,7 @@ HgInfo exportMercurial(ref store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -93,17 +93,14 @@ HgInfo exportMercurial(ref store, const std::string & uri, try { runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); } - catch (ExecError & e){ + catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) - { + if (pathExists(transJournal)) { runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - else - { - throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 5c1705e2f..571c194ec 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -822,7 +822,7 @@ CachedDownloadResult Downloader::downloadCached( auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; result.effectiveUri = request.uri; result.etag = ss[1]; From 4f6a7c86218c748df2a8700eca8ba9edbedd9ca5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 15:17:31 +0200 Subject: [PATCH 0336/2548] shell.nix: Unset $PYTHONPATH It breaks gdb. --- shell.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.nix b/shell.nix index dd2950b29..34abf05ea 100644 --- a/shell.nix +++ b/shell.nix @@ -23,5 +23,6 @@ with import ./release-common.nix { inherit pkgs; }; configureFlags+=" --prefix=$prefix" PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH PATH=$prefix/bin:$PATH + unset PYTHONPATH ''; } From d132d057a85aa1812c4133feed6c9b34ca70671d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 15:29:05 +0200 Subject: [PATCH 0337/2548] Handle store symlinks in flake directories E.g. 'nix path-info ./result' inside a flake directory now works again. --- src/libexpr/flake/flakeref.cc | 8 ++++++++ src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 1 + src/nix/installables.cc | 22 +++++++++++----------- tests/flakes.sh | 4 ++++ 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 24af09124..7631cd53a 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -143,6 +143,14 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) IsPath d; if (allowRelative) { d.path = absPath(uri); + try { + if (!S_ISDIR(lstat(d.path).st_mode)) + throw BadFlakeRef("path '%s' is not a flake (sub)directory"); + } catch (SysError & e) { + if (e.errNo == ENOENT || e.errNo == EISDIR) + throw BadFlakeRef("flake '%s' does not exist"); + throw; + } while (true) { if (pathExists(d.path + "/.git")) break; subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fabeeea4..c5a771030 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -55,7 +55,7 @@ Path Store::followLinksToStore(const Path & _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0f..558ea79af 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -26,6 +26,7 @@ MakeError(InvalidPath, Error) MakeError(Unsupported, Error) MakeError(SubstituteGone, Error) MakeError(SubstituterDisabled, Error) +MakeError(NotInStore, Error) struct BasicDerivation; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index feaf57f0c..d43f86c0c 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -394,9 +394,18 @@ std::vector> SourceExprCommand::parseInstallables( } else { + auto follow = [&](const std::string & s) -> std::optional { + try { + return store->followLinksToStorePath(s); + } catch (NotInStore &) { + return {}; + } + }; + for (auto & s : ss) { size_t colon; + std::optional storePath; if (s.compare(0, 1, "(") == 0) result.push_back(std::make_shared(*this, s)); @@ -422,17 +431,8 @@ std::vector> SourceExprCommand::parseInstallables( getDefaultFlakeAttrPathPrefixes())); } - else if (s.find('/') != std::string::npos || s == ".") { - Path storePath; - try { - storePath = store->toStorePath(store->followLinksToStore(s)); - } catch (Error) { } - if (storePath != "") - result.push_back(std::make_shared(storePath)); - else - result.push_back(std::make_shared(*this, FlakeRef(s, true), - getDefaultFlakeAttrPaths())); - } + else if (s.find('/') != std::string::npos && (storePath = follow(s))) + result.push_back(std::make_shared(*storePath)); else throw Error("unsupported argument '%s'", s); diff --git a/tests/flakes.sh b/tests/flakes.sh index 0571c2a62..bfe00a674 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -149,6 +149,10 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake1 nix build -o $TEST_ROOT/result --flake-registry $registry $flake1Dir nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake1Dir +# CHeck that store symlinks inside a flake are not interpreted as flakes. +nix build -o $flake1Dir/result --flake-registry $registry file://$flake1Dir +nix path-info $flake1Dir/result + # Building a flake with an unlocked dependency should fail in pure mode. (! nix eval "(builtins.getFlake "$flake2Dir")") From aa2846198f46c1260a91a6bf21a2f53997f1f274 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 18:34:43 +0200 Subject: [PATCH 0338/2548] Don't update the global registry when building a locked flake It's unnecessary and slows things down (e.g. when you're on a Thalys with super-crappy Internet). --- src/libexpr/flake/flake.cc | 4 +++- tests/flakes.sh | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 215eb85b6..e9db9d80e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -119,7 +119,9 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) { FlakeRef resolvedRef = lookupFlake(state, flakeRef, - impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + impureIsAllowed && !flakeRef.isDirect() + ? state.getFlakeRegistries() + : std::vector>()); if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); diff --git a/tests/flakes.sh b/tests/flakes.sh index bfe00a674..ccab84612 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -168,6 +168,9 @@ git -C $flake2Dir commit flake.lock -m 'Add flake.lock' nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar [[ -z $(git -C $flake2Dir diff master) ]] +# Building with a lockfile should not require a fetch of the registry. +nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir:bar --tarball-ttl 0 + # Updating the flake should not change the lockfile. nix flake update --flake-registry $registry $flake2Dir [[ -z $(git -C $flake2Dir diff master) ]] @@ -297,7 +300,7 @@ nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy flake3:fn # Test doing multiple `lookupFlake`s nix build -o $TEST_ROOT/result --flake-registry $registry flake4:xyzzy -nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake4Dir:xyzzy +#nix build -o $TEST_ROOT/result --flake-registry $registry file://$flake4Dir:xyzzy # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C $flake3Dir checkout -b removeXyzzy From d4fe9daed6f48ebdcea18a1952cbecd30a846e70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 19:04:58 +0200 Subject: [PATCH 0339/2548] Simplify getFlake() / fetchFlake() logic --- src/libexpr/flake/flake.cc | 50 ++++++++++++++++++++++---------------- src/libexpr/flake/flake.hh | 9 ++++++- src/nix/flake.cc | 7 +++--- tests/flakes.sh | 14 ++++------- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index e9db9d80e..302549a3c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -89,9 +89,6 @@ FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Regist static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, std::vector pastSearches) { - if (registries.empty() && !flakeRef.isDirect()) - throw Error("indirect flake reference '%s' is not allowed", flakeRef); - for (std::shared_ptr registry : registries) { auto i = registry->entries.find(flakeRef); if (i != registry->entries.end()) { @@ -115,16 +112,24 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const return flakeRef; } -// Lookups happen here too -static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +FlakeRef maybeLookupFlake( + EvalState & state, + const FlakeRef & flakeRef, + bool allowLookup) { - FlakeRef resolvedRef = lookupFlake(state, flakeRef, - impureIsAllowed && !flakeRef.isDirect() - ? state.getFlakeRegistries() - : std::vector>()); + if (!flakeRef.isDirect()) { + if (allowLookup) + return lookupFlake(state, flakeRef, state.getFlakeRegistries()); + else + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef); + } else + return flakeRef; +} - if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) - throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & resolvedRef) +{ + assert(resolvedRef.isDirect()); auto doGit = [&](const GitInfo & gitInfo) { FlakeRef ref(resolvedRef.baseRef()); @@ -190,10 +195,9 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool else abort(); } -// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { - SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + SourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; @@ -278,10 +282,9 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowe return flake; } -// Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef) { - auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + auto sourceInfo = fetchFlake(state, flakeRef); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; @@ -347,7 +350,7 @@ static std::pair updateLocks( } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); + auto nonFlake = getNonFlake(state, maybeLookupFlake(state, ref, allowedToUseRegistries(handleLockFile, false))); newEntry.nonFlakeInputs.insert_or_assign(id, NonFlakeInput( nonFlake.sourceInfo.resolvedRef, @@ -364,7 +367,7 @@ static std::pair updateLocks( throw Error("cannot update flake dependency '%s' in pure mode", inputRef); newEntry.flakeInputs.insert_or_assign(inputRef, updateLocks(state, - getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + getFlake(state, maybeLookupFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false))), handleLockFile, {}, false).second); } } @@ -376,7 +379,7 @@ static std::pair updateLocks( and optionally write it to file, it the flake is writable. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { - auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + auto flake = getFlake(state, maybeLookupFlake(state, topRef, allowedToUseRegistries(handleLockFile, true))); LockFile oldLockFile; @@ -441,7 +444,10 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto lazyFlake = (FlakeInput *) args[0]->attrs; - auto flake = getFlake(state, lazyFlake->ref, false); + + assert(lazyFlake->ref.isImmutable()); + + auto flake = getFlake(state, lazyFlake->ref); if (flake.sourceInfo.narHash != lazyFlake->narHash) throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); @@ -453,6 +459,8 @@ static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args { auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + assert(lazyNonFlake->ref.isImmutable()); + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 81b6541f0..de0feb2c4 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -81,7 +81,14 @@ struct NonFlake : originalRef(origRef), sourceInfo(sourceInfo) {}; }; -Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); +Flake getFlake(EvalState &, const FlakeRef &); + +/* If 'allowLookup' is true, then resolve 'flakeRef' using the + registries. */ +FlakeRef maybeLookupFlake( + EvalState & state, + const FlakeRef & flakeRef, + bool allowLookup); /* Fingerprint of a locked flake; used as a cache key. */ typedef Hash Fingerprint; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 91c6b4276..49f7c33c7 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -38,7 +38,8 @@ public: Flake getFlake() { auto evalState = getEvalState(); - return flake::getFlake(*evalState, getFlakeRef(), useRegistries); + return flake::getFlake(*evalState, + maybeLookupFlake(*evalState, getFlakeRef(), useRegistries)); } ResolvedFlake resolveFlake() @@ -425,13 +426,13 @@ struct CmdFlakePin : virtual Args, EvalCommand FlakeRegistry userRegistry = *readRegistry(userRegistryPath); auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; + it->second = getFlake(*evalState, maybeLookupFlake(*evalState, it->second, true)).sourceInfo.resolvedRef; writeRegistry(userRegistry, userRegistryPath); } else { std::shared_ptr globalReg = evalState->getGlobalFlakeRegistry(); it = globalReg->entries.find(FlakeRef(alias)); if (it != globalReg->entries.end()) { - auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; + auto newRef = getFlake(*evalState, maybeLookupFlake(*evalState, it->second, true)).sourceInfo.resolvedRef; userRegistry.entries.insert_or_assign(alias, newRef); writeRegistry(userRegistry, userRegistryPath); } else diff --git a/tests/flakes.sh b/tests/flakes.sh index ccab84612..d2b168712 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -107,9 +107,6 @@ cat > $registry < $registry < Date: Sun, 23 Jun 2019 22:19:14 +0200 Subject: [PATCH 0340/2548] nix doctor: Fix typo --- src/nix/doctor.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index f2cf04758..98260127b 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -116,4 +116,4 @@ struct CmdDoctor : StoreCommand } }; -static auto r1 = registerCommand("doctore"); +static auto r1 = registerCommand("doctor"); From 99cec651c9c1d95656e182871f0bb186f798077c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 16:52:11 +0200 Subject: [PATCH 0341/2548] Add more fromTOML tests --- tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 392ff7a72..3c16e1fa1 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 5626ef338..dd9ad9ea1 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -147,6 +147,25 @@ name = "Nail" sku = 284758393 color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" '') (builtins.fromTOML '' From 94f11d0a61bba15ff8a93f2de8d1883783c8c508 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 17:09:09 +0200 Subject: [PATCH 0342/2548] Fix abort in fromTOML Fixes #2969. --- src/libexpr/primops/fromTOML.cc | 13 +++++++++++++ tests/lang/eval-okay-fromTOML.exp | 2 +- tests/lang/eval-okay-fromTOML.nix | 7 +++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4128de05d..a84e569e9 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); } + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + else if (t->is_value()) { if (auto val = t->as()) mkInt(v, val->get()); diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 3c16e1fa1..d0dd3af2c 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index dd9ad9ea1..963932689 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -198,4 +198,11 @@ "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + ] From 15fa70cd1b853f5e62662b99ccb9ef3da6cfadff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:06:37 +0200 Subject: [PATCH 0343/2548] Downloader: Propagate exceptions from decompressionSink->finish() --- src/libstore/download.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 571c194ec..70cfaf0dd 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -311,8 +311,13 @@ struct CurlDownloader : public Downloader debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; From a67cf5a3585c41dd9f219a2c7aa9cf67fa69520b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:48:52 +0200 Subject: [PATCH 0344/2548] Fix 'error 9 while decompressing xz file' Once we've started writing data to a Sink, we can't restart a download request, because then we end up writing duplicate data to the Sink. Therefore we shouldn't handle retries in Downloader but at a higher level (in particular, in copyStorePath()). Fixes #2952. --- src/libstore/binary-cache-store.cc | 18 +++--- src/libstore/download.cc | 67 ++++---------------- src/libstore/download.hh | 11 +++- src/libstore/http-binary-cache-store.cc | 55 +++++++++++----- src/libstore/store-api.cc | 84 +++++++++++++------------ src/libutil/retry.hh | 38 +++++++++++ src/libutil/types.hh | 2 + 7 files changed, 156 insertions(+), 119 deletions(-) create mode 100644 src/libutil/retry.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..8b736056e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,8 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "retry.hh" +#include "download.hh" #include @@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; + return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + }); } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 70cfaf0dd..cf79b2af5 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "retry.hh" #ifdef ENABLE_S3 #include @@ -19,11 +20,9 @@ #include #include -#include #include #include #include -#include #include using namespace std::string_literals; @@ -46,9 +45,6 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; - std::random_device rd; - std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -61,12 +57,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; std::string encoding; @@ -385,9 +375,7 @@ struct CurlDownloader : public Downloader } } - attempt++; - - auto exc = + fail( code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -398,31 +386,15 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); + request.verb(), request.uri, curl_easy_strerror(code), code))); } } }; struct State { - struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { - return i1->embargo > i2->embargo; - } - }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::vector> incoming; }; Sync state_; @@ -435,7 +407,6 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() - : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -529,9 +500,7 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ + /* Add new curl requests from the incoming requests queue. */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -540,22 +509,9 @@ struct CurlDownloader : public Downloader } std::vector> incoming; - auto now = std::chrono::steady_clock::now(); - { auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } + std::swap(state->incoming, incoming); quit = state->quit; } @@ -582,7 +538,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); + state->incoming.clear(); state->quit = true; } } @@ -598,7 +554,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); + state->incoming.push_back(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -681,7 +637,9 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return enqueueDownload(request).get(); + return retry(request.tries, [&]() { + return enqueueDownload(request).get(); + }); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -868,7 +826,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("%s; using cached result", e.msg()); result.etag = expectedETag; } } @@ -931,5 +889,4 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } - } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c095ad053..7548b83ae 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -99,11 +99,13 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. */ + /* Synchronously download a file. The request will be retried in + case of transient failures. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + invoked on the thread of the caller. The request will not be + retried in case of transient failures. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -129,6 +131,11 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } + + bool isTransient() override + { + return error == Downloader::Error::Transient; + } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 11c34fdac..ff2c10354 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "retry.hh" namespace nix { @@ -113,7 +114,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -136,21 +136,46 @@ protected: { checkEnabled(); - auto request(makeRequest(path)); + struct State + { + DownloadRequest request; + std::function tryDownload; + unsigned int attempt = 0; + State(DownloadRequest && request) : request(request) {} + }; - getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - maybeDisable(); - callback.rethrow(); - } catch (...) { - callback.rethrow(); - } - }}); + auto state = std::make_shared(makeRequest(path)); + + state->tryDownload = [callback, state, this]() { + getDownloader()->enqueueDownload(state->request, + {[callback, state, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + ++state->attempt; + if (state->attempt < state->request.tries && e.isTransient()) { + auto ms = retrySleepTime(state->attempt); + warn("%s; retrying in %d ms", e.what(), ms); + /* We can't sleep here because that would + block the download thread. So use a + separate thread for sleeping. */ + std::thread([state, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + state->tryDownload(); + }).detach(); + } else { + maybeDisable(); + callback.rethrow(); + } + } catch (...) { + callback.rethrow(); + } + }}); + }; + + state->tryDownload(); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c5a771030..f577799da 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,10 +6,11 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "retry.hh" +#include "download.hh" #include - namespace nix { @@ -572,54 +573,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + retry(downloadSettings.tries, [&]() { - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); - auto info = srcStore->queryPathInfo(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); - uint64_t total = 0; + auto info = srcStore->queryPathInfo(storePath); - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; + uint64_t total = 0; - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); + }); } diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh new file mode 100644 index 000000000..b45cb37f7 --- /dev/null +++ b/src/libutil/retry.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "logging.hh" + +#include +#include +#include +#include + +namespace nix { + +inline unsigned int retrySleepTime(unsigned int attempt) +{ + std::random_device rd; + std::mt19937 mt19937; + return 250.0 * std::pow(2.0f, + attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); +} + +template +C retry(unsigned int attempts, std::function && f) +{ + unsigned int attempt = 0; + while (true) { + try { + return f(); + } catch (BaseError & e) { + ++attempt; + if (attempt >= attempts || !e.isTransient()) + throw; + auto ms = retrySleepTime(attempt); + warn("%s; retrying in %d ms", e.what(), ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 92bf469b5..88e3243f4 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,6 +109,8 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); + + virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ From dc29e9fb47f9f98a851dc88b2bd3cae4b5c1fe6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Feb 2019 23:20:50 +0800 Subject: [PATCH 0345/2548] downloadCached: Return ETag (cherry picked from commit 529add316c5356a8060c35f987643b7bf5c796dc) --- src/libexpr/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 2 +- src/libstore/download.cc | 17 +++++++++++++---- src/libstore/download.hh | 12 ++++++++++-- src/nix-channel/nix-channel.cc | 6 +++--- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..37c74a94b 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -46,7 +46,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); + return getDownloader()->downloadCached(state.store, s, true).path; else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 78a503907..29860fb05 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -677,7 +677,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 06f577f36..55a1bde11 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2083,7 +2083,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 22382ab1d..cb77cdc77 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -790,7 +790,7 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) { auto url = resolveUri(url_); @@ -802,8 +802,11 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa Path expectedStorePath; if (expectedHash) { expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -822,6 +825,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -833,6 +838,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa skip = true; if (effectiveUrl) *effectiveUrl = url_; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -850,6 +856,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa auto res = download(request); if (effectiveUrl) *effectiveUrl = res.effectiveUrl; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; @@ -871,6 +878,7 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa } catch (DownloadError & e) { if (storePath.empty()) throw; printError(format("warning: %1%; using cached result") % e.msg()); + result.etag = expectedETag; } } @@ -904,7 +912,8 @@ Path Downloader::downloadCached(ref store, const string & url_, bool unpa url, expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..8acfe4e1a 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -41,6 +41,12 @@ struct DownloadResult uint64_t bodySize = 0; }; +struct CachedDownloadResult +{ + Path path; + std::optional etag; +}; + class Store; struct Downloader @@ -64,8 +70,10 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached( + ref store, const string & uri, bool unpack, string name = "", + const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, + int ttl = settings.tarballTtl); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..7b23088a2 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -88,7 +88,7 @@ static void update(const StringSet & channelNames) // definition from a consistent location if the redirect changes mid-download. std::string effectiveUrl; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); + auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; url = chomp(std::move(effectiveUrl)); // If the URL contains a version number, append it to the name @@ -123,10 +123,10 @@ static void update(const StringSet & channelNames) // Download the channel tarball. auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } catch (DownloadError & e) { fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, fullURL, false).path; } chomp(filename); } From b43e1e186e50bedc30d05d063dffbd751c1ff161 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 May 2019 15:38:24 +0200 Subject: [PATCH 0346/2548] CachedDownloadResult: Include store path Also, make fetchGit and fetchMercurial update allowedPaths properly. (Maybe the evaluator, rather than the caller of the evaluator, should apply toRealPath(), but that's a bigger change.) (cherry picked from commit 5c34d665386f4053d666b0899ecca0639e500fbd) --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 2 +- src/libstore/download.cc | 2 ++ src/libstore/download.hh | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index aaf02c856..cf7ccca41 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -235,7 +235,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374..d576138f5 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -214,7 +214,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cb77cdc77..975cfd97d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -804,6 +804,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; + result.storePath = expectedStorePath; result.path = store->toRealPath(expectedStorePath); return result; } @@ -912,6 +913,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & url, expectedHash.to_string(), gotHash.to_string()); } + result.storePath = storePath; result.path = store->toRealPath(storePath); return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 8acfe4e1a..aa8c34be2 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -43,6 +43,9 @@ struct DownloadResult struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; Path path; std::optional etag; }; From 2fef4dd29673ef383b695480c9b8212f2e8d4711 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:06:37 +0200 Subject: [PATCH 0347/2548] Downloader: Propagate exceptions from decompressionSink->finish() (cherry picked from commit 15fa70cd1b853f5e62662b99ccb9ef3da6cfadff) --- src/libstore/download.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 975cfd97d..8e94646d4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -327,8 +327,13 @@ struct CurlDownloader : public Downloader debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; From 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Jun 2019 21:48:52 +0200 Subject: [PATCH 0348/2548] Fix 'error 9 while decompressing xz file' Once we've started writing data to a Sink, we can't restart a download request, because then we end up writing duplicate data to the Sink. Therefore we shouldn't handle retries in Downloader but at a higher level (in particular, in copyStorePath()). Fixes #2952. (cherry picked from commit a67cf5a3585c41dd9f219a2c7aa9cf67fa69520b) --- src/libstore/binary-cache-store.cc | 18 +++--- src/libstore/download.cc | 67 ++++---------------- src/libstore/download.hh | 11 +++- src/libstore/http-binary-cache-store.cc | 55 +++++++++++----- src/libstore/store-api.cc | 84 +++++++++++++------------ src/libutil/retry.hh | 38 +++++++++++ src/libutil/types.hh | 2 + 7 files changed, 156 insertions(+), 119 deletions(-) create mode 100644 src/libutil/retry.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..8b736056e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,6 +10,8 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "retry.hh" +#include "download.hh" #include @@ -79,13 +81,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; + return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; + }); } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 8e94646d4..43d231e30 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,6 +8,7 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" +#include "retry.hh" #ifdef ENABLE_S3 #include @@ -19,11 +20,9 @@ #include #include -#include #include #include #include -#include #include using namespace std::string_literals; @@ -62,9 +61,6 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; - std::random_device rd; - std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -77,12 +73,6 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; - unsigned int attempt = 0; - - /* Don't start this download until the specified time point - has been reached. */ - std::chrono::steady_clock::time_point embargo; - struct curl_slist * requestHeaders = 0; std::string encoding; @@ -401,9 +391,7 @@ struct CurlDownloader : public Downloader } } - attempt++; - - auto exc = + fail( code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -414,31 +402,15 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); - - /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); - embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); - } - else - fail(exc); + request.verb(), request.uri, curl_easy_strerror(code), code))); } } }; struct State { - struct EmbargoComparator { - bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { - return i1->embargo > i2->embargo; - } - }; bool quit = false; - std::priority_queue, std::vector>, EmbargoComparator> incoming; + std::vector> incoming; }; Sync state_; @@ -451,7 +423,6 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() - : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -545,9 +516,7 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue, - except for requests that are embargoed (waiting for a - retry timeout to expire). */ + /* Add new curl requests from the incoming requests queue. */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -556,22 +525,9 @@ struct CurlDownloader : public Downloader } std::vector> incoming; - auto now = std::chrono::steady_clock::now(); - { auto state(state_.lock()); - while (!state->incoming.empty()) { - auto item = state->incoming.top(); - if (item->embargo <= now) { - incoming.push_back(item); - state->incoming.pop(); - } else { - if (nextWakeup == std::chrono::steady_clock::time_point() - || item->embargo < nextWakeup) - nextWakeup = item->embargo; - break; - } - } + std::swap(state->incoming, incoming); quit = state->quit; } @@ -598,7 +554,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - while (!state->incoming.empty()) state->incoming.pop(); + state->incoming.clear(); state->quit = true; } } @@ -614,7 +570,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push(item); + state->incoming.push_back(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -697,7 +653,9 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return enqueueDownload(request).get(); + return retry(request.tries, [&]() { + return enqueueDownload(request).get(); + }); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -883,7 +841,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("%s; using cached result", e.msg()); result.etag = expectedETag; } } @@ -933,5 +891,4 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } - } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index aa8c34be2..752f91857 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -62,11 +62,13 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. */ + /* Synchronously download a file. The request will be retried in + case of transient failures. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + invoked on the thread of the caller. The request will not be + retried in case of transient failures. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -95,6 +97,11 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } + + bool isTransient() override + { + return error == Downloader::Error::Transient; + } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..5133dba57 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "retry.hh" namespace nix { @@ -114,7 +115,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -137,21 +137,46 @@ protected: { checkEnabled(); - auto request(makeRequest(path)); + struct State + { + DownloadRequest request; + std::function tryDownload; + unsigned int attempt = 0; + State(DownloadRequest && request) : request(request) {} + }; - getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - maybeDisable(); - callback.rethrow(); - } catch (...) { - callback.rethrow(); - } - }}); + auto state = std::make_shared(makeRequest(path)); + + state->tryDownload = [callback, state, this]() { + getDownloader()->enqueueDownload(state->request, + {[callback, state, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + ++state->attempt; + if (state->attempt < state->request.tries && e.isTransient()) { + auto ms = retrySleepTime(state->attempt); + warn("%s; retrying in %d ms", e.what(), ms); + /* We can't sleep here because that would + block the download thread. So use a + separate thread for sleeping. */ + std::thread([state, ms]() { + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + state->tryDownload(); + }).detach(); + } else { + maybeDisable(); + callback.rethrow(); + } + } catch (...) { + callback.rethrow(); + } + }}); + }; + + state->tryDownload(); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..28ad7c019 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,10 +6,11 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "retry.hh" +#include "download.hh" #include - namespace nix { @@ -572,54 +573,57 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + retry(downloadSettings.tries, [&]() { - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); - auto info = srcStore->queryPathInfo(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); - uint64_t total = 0; + auto info = srcStore->queryPathInfo(storePath); - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; + uint64_t total = 0; - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); - }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore->addToStore(*info, *source, repair, checkSigs); + }); } diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh new file mode 100644 index 000000000..b45cb37f7 --- /dev/null +++ b/src/libutil/retry.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "logging.hh" + +#include +#include +#include +#include + +namespace nix { + +inline unsigned int retrySleepTime(unsigned int attempt) +{ + std::random_device rd; + std::mt19937 mt19937; + return 250.0 * std::pow(2.0f, + attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); +} + +template +C retry(unsigned int attempts, std::function && f) +{ + unsigned int attempt = 0; + while (true) { + try { + return f(); + } catch (BaseError & e) { + ++attempt; + if (attempt >= attempts || !e.isTransient()) + throw; + auto ms = retrySleepTime(attempt); + warn("%s; retrying in %d ms", e.what(), ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 92bf469b5..88e3243f4 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,6 +109,8 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); + + virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ From 7b9c68766d513260d5262d5782b46384834cdb33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 08:43:45 +0200 Subject: [PATCH 0349/2548] Add '--no-net' convenience flag This flag * Disables substituters. * Sets the tarball-ttl to infinity (ensuring e.g. that the flake registry and any downloaded flakes are considered current). * Disables retrying downloads and sets the connection timeout to the minimum. (So it doesn't completely disable downloads at the moment.) (cherry picked from commit 8ea842260b4fd93315d35c5ba94b1ff99ab391d8) --- src/libmain/common-args.cc | 11 +++++++++++ src/libstore/download.cc | 18 +----------------- src/libstore/download.hh | 23 ++++++++++++++++++++++- src/libstore/globals.hh | 2 +- src/libstore/http-binary-cache-store.cc | 1 - 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9e1d7cee6..0486932b2 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "globals.hh" +#include "download.hh" namespace nix { @@ -44,6 +45,16 @@ MixCommonArgs::MixCommonArgs(const string & programName) settings.set("max-jobs", s); }); + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([]() { + settings.useSubstitutes = false; + settings.tarballTtl = std::numeric_limits::max(); + downloadSettings.tries = 0; + downloadSettings.connectTimeout = 1; + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 43d231e30..342a8aa21 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -29,23 +29,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 752f91857..dae082ab9 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,34 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..0af8215d1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -255,7 +255,7 @@ public: "Secret keys with which to sign local builds."}; Setting tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5133dba57..5633b4355 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -85,7 +85,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { From f8b30338ac231262bdf19844f044f0572c460048 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 23:36:29 +0200 Subject: [PATCH 0350/2548] Refactor downloadCached() interface (cherry picked from commit df3f5a78d5ab0a1f2dc9d288b271b38a9b8b33b5) --- src/libexpr/common-eval-args.cc | 8 +++--- src/libexpr/parser.y | 4 ++- src/libexpr/primops.cc | 22 ++++++++--------- src/libstore/download.cc | 44 ++++++++++++++++----------------- src/libstore/download.hh | 20 +++++++++++---- src/nix-channel/nix-channel.cc | 14 +++++------ 6 files changed, 63 insertions(+), 49 deletions(-) diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 37c74a94b..13950ab8d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true).path; - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 29860fb05..967c88d9b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -677,7 +677,9 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true).path }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 55a1bde11..070e72f3a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2050,9 +2050,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,27 +2063,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash).path; + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 342a8aa21..892c1b21d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -293,10 +293,10 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); @@ -737,18 +737,20 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached(ref store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); if (store->isValidPath(expectedStorePath)) { CachedDownloadResult result; result.storePath = expectedStorePath; @@ -782,10 +784,9 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); @@ -799,18 +800,17 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -830,7 +830,7 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -853,11 +853,11 @@ CachedDownloadResult Downloader::downloadCached(ref store, const string & } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + url, request.expectedHash.to_string(), gotHash.to_string()); } result.storePath = storePath; diff --git a/src/libstore/download.hh b/src/libstore/download.hh index dae082ab9..9e965b506 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -57,11 +57,23 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + struct CachedDownloadResult { // Note: 'storePath' may be different from 'path' when using a @@ -69,6 +81,7 @@ struct CachedDownloadResult Path storePath; Path path; std::optional etag; + std::string effectiveUri; }; class Store; @@ -96,10 +109,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached( - ref store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, - int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 7b23088a2..bd1371dba 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0).path; - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -121,12 +123,10 @@ static void update(const StringSet & channelNames) } // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false).path; + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } From 64ec087f582cec33733f4102ab42a3e8f291758a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2019 12:49:18 +0200 Subject: [PATCH 0351/2548] Fix 32-bit overflow with --no-net --no-net causes tarballTtl to be set to the largest 32-bit integer, which causes comparison like 'time + tarballTtl < other_time' to fail on 32-bit systems. So cast them to 64-bit first. https://hydra.nixos.org/build/95076624 (cherry picked from commit 29ccb2e9697ee2184012dd13854e487928ae4441) --- src/libexpr/primops/fetchGit.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 13 +++++-------- src/libstore/download.cc | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index cf7ccca41..3dcf3e9ff 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -116,7 +116,7 @@ GitInfo exportGit(ref store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d576138f5..a907d0e1c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -96,17 +96,14 @@ HgInfo exportMercurial(ref store, const std::string & uri, try { runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); } - catch (ExecError & e){ + catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) - { + if (pathExists(transJournal)) { runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - else - { - throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 892c1b21d..6ce9525c3 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -784,7 +784,7 @@ CachedDownloadResult Downloader::downloadCached( auto ss = tokenizeString>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + request.ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; result.effectiveUri = request.uri; result.etag = ss[1]; From 5600b070a766b83200a68d3632793917cf19a550 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:12:03 +0200 Subject: [PATCH 0352/2548] Add "warning" verbosity level This ensures that "nix" shows warnings. Previously these were hidden because they were at "info" level. (cherry picked from commit 615a9d031d22a6aee64f8511e15685e47b6f8796) --- src/libutil/logging.cc | 3 ++- src/libutil/logging.hh | 1 + src/nix/main.cc | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae..b379306f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102..5f2219445 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, diff --git a/src/nix/main.cc b/src/nix/main.cc index 25e321b86..19bb7a543 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -93,7 +93,7 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlError; + verbosity = lvlWarn; settings.verboseBuild = false; NixArgs args; From 09dde33c19129b949a064d35e248a3e318178b08 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Jun 2019 09:57:22 +0200 Subject: [PATCH 0353/2548] Automatically use --no-net if there are no network interfaces (cherry picked from commit 04a59769963fe2a28d10ba15de743fe499333c80) --- src/libmain/common-args.cc | 11 -------- src/nix/main.cc | 55 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 0486932b2..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,5 @@ #include "common-args.hh" #include "globals.hh" -#include "download.hh" namespace nix { @@ -45,16 +44,6 @@ MixCommonArgs::MixCommonArgs(const string & programName) settings.set("max-jobs", s); }); - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([]() { - settings.useSubstitutes = false; - settings.tarballTtl = std::numeric_limits::max(); - downloadSettings.tries = 0; - downloadSettings.connectTimeout = 1; - }); - std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/nix/main.cc b/src/nix/main.cc index 19bb7a543..73c4b8db1 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,19 +8,52 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include +#include +#include +#include + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) && + !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; + bool useNet = true; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -53,6 +86,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -108,6 +146,23 @@ void mainWrapped(int argc, char * * argv) startProgressBar(args.printBuildLogs); + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + args.command->prepare(); args.command->run(); } From 88571219d97f2bdfdbafcff25ef6ee424b0b008f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jun 2019 13:27:16 +0200 Subject: [PATCH 0354/2548] nix-channel: Don't fetch binary-cache-url This has been ignored since the Perl->C++ rewrite. --- corepkgs/unpack-channel.nix | 8 ++------ doc/manual/command-ref/nix-channel.xml | 23 +++++------------------ src/libstore/store-api.hh | 3 +-- src/nix-channel/nix-channel.cc | 9 --------- 4 files changed, 8 insertions(+), 35 deletions(-) diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index a654db40e..d39a20637 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -18,21 +18,17 @@ let if [ * != $channelName ]; then mv * $out/$channelName fi - if [ -n "$binaryCacheURL" ]; then - mkdir $out/binary-caches - echo -n "$binaryCacheURL" > $out/binary-caches/$channelName - fi ''; in -{ name, channelName, src, binaryCacheURL ? "" }: +{ name, channelName, src }: derivation { system = builtins.currentSystem; builder = shell; args = [ "-e" builder ]; - inherit name channelName src binaryCacheURL; + inherit name channelName src; PATH = "${nixBinDir}:${coreutils}"; diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a76..5a2866e6b 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,12 +31,11 @@ Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing both a set of Nix -expressions and a pointer to a binary cache. See also . +A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. See also . This command has the following operations: @@ -172,18 +171,6 @@ following files: - binary-cache-url - - A file containing the URL to a binary cache (such - as https://cache.nixos.org). Nix will automatically - check this cache for pre-built binaries, if the user has - sufficient rights to add binary caches. For instance, in a - multi-user Nix setup, the binary caches provided by the channels - of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored. - - - diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0f..599677376 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -766,8 +766,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list> getDefaultSubstituters(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index bd1371dba..06eb3d23b 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -113,15 +113,6 @@ static void update(const StringSet & channelNames) } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. try { filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; From 324a5dc92f8e50e6b637c5e67dea48c80be10837 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Jun 2019 21:57:18 +0200 Subject: [PATCH 0355/2548] ProgressBar: Fix updating 'updateCV.notify_one()' does nothing if the update thread is not waiting for updateCV (in particular this happens when it is sleeping on quitCV). So also set a variable to ensure that the update isn't lost. --- src/nix/progress-bar.cc | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index e71045408..b1c1d87de 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -62,6 +62,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool haveUpdate = true; }; Sync state_; @@ -83,7 +84,8 @@ public: updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { - state.wait(updateCV); + if (!state->haveUpdate) + state.wait(updateCV); draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } @@ -178,7 +180,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(); + update(*state); } /* Check whether an activity has an ancestore with the specified @@ -213,7 +215,7 @@ public: state->its.erase(i); } - update(); + update(*state); } void result(ActivityId act, ResultType type, const std::vector & fields) override @@ -223,7 +225,7 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(); + update(*state); } else if (type == resBuildLogLine) { @@ -239,26 +241,26 @@ public: info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(); + update(*state); } } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(); + update(*state); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(); + update(*state); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(); + update(*state); } else if (type == resProgress) { @@ -269,7 +271,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(); + update(*state); } else if (type == resSetExpected) { @@ -281,17 +283,19 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(); + update(*state); } } - void update() + void update(State & state) { + state.haveUpdate = true; updateCV.notify_one(); } void draw(State & state) { + state.haveUpdate = false; if (!state.active) return; std::string line; From ec0087df0a2da5c68363b3f4509d4545deed97a0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 27 Jun 2019 14:22:53 -0400 Subject: [PATCH 0356/2548] =?UTF-8?q?Don=E2=80=99t=20use=20entire=20/etc/n?= =?UTF-8?q?sswitch.conf=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default nsswitch.conf(5) file in most distros can handle many different things including host name, user names, groups, etc. In Nix, we want to limit the amount of impurities that come from these things. As a result, we should only allow nss to be used for gethostbyname(3) and getservent(3). /cc @Ericson2314 --- src/libstore/build.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5b38bcf3c..813d7e2c2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2728,7 +2728,13 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); if (pathExists("/var/run/nscd/socket")) From 6847c9278840edc97f1ef85b5fafac4338fb1b37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Jun 2019 15:38:23 +0200 Subject: [PATCH 0357/2548] Fix macOS build failure Issue #2976. --- src/nix/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 73c4b8db1..a80fd0ea6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -39,8 +39,8 @@ static bool haveInternet() return true; } } else if (i->ifa_addr->sa_family == AF_INET6) { - if (!IN6_IS_ADDR_LOOPBACK(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr) && - !IN6_IS_ADDR_LINKLOCAL(((sockaddr_in6 *) i->ifa_addr)->sin6_addr.s6_addr)) + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) return true; } } From ec58ba38c55a3ab61fdb1da7d746251e4887ea2c Mon Sep 17 00:00:00 2001 From: Aniket Deshpande Date: Fri, 28 Jun 2019 19:43:22 +0530 Subject: [PATCH 0358/2548] Fix `http2 = false` having no effect. Fixes #2971. Setting `http2 = false` in nix config (e.g. /etc/nix/nix.conf) had no effect, and `nix-env -vvvvv -i hello` still downloaded .nar packages using HTTP/2. In `src/libstore/download.cc`, the `CURL_HTTP_VERSION_2TLS` option was being explicitly set when `downloadSettings.enableHttp2` was `true`, but, `CURL_HTTP_VERSION_1_1` option was not being explicitly set when `downloadSettings.enableHttp2` was `false`. This may be because `https://curl.haxx.se/libcurl/c/libcurl-env.html` states: "You have to set this option if you want to use libcurl's HTTP/2 support." but, also, in the changelog, states: "DEFAULT Since curl 7.62.0: CURL_HTTP_VERSION_2TLS Before that: CURL_HTTP_VERSION_1_1" So, the default setting for `libcurl` is HTTP/2 for version >= 7.62.0. In this commit, option `CURLOPT_HTTP_VERSION` is explicitly set to `CURL_HTTP_VERSION_1_1` when `downloadSettings.enableHttp2` nix config setting is `false`. This can be tested by running `nix-env -vvvvv -i hello | grep HTTP` --- src/libstore/download.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 6ce9525c3..0c5a73ea3 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -244,6 +244,8 @@ struct CurlDownloader : public Downloader #if LIBCURL_VERSION_NUM >= 0x072f00 if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); From 97baf32fbca13101f58d30bb3a601302c3d560f3 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Tue, 2 Jul 2019 00:12:12 +0200 Subject: [PATCH 0359/2548] build: add exit code for hash and check mismatches Makes it easier to identify the failure reason in other tooling, eg. differentiate between a non-deterministic --check vs a failed build. $ nix-build '' --argstr url http://example.org --argstr sha256 0000000000000000000000000000000000000000000000000000 hash mismatch in fixed-output derivation '/nix/store/nzi9ck45rwlxzcwr25is7qlf3hs5xl83-example.org': wanted: sha256:0000000000000000000000000000000000000000000000000000 got: sha256:08y4734bm2zahw75b16bcmcg587vvyvh0n11gwiyir70divwp1rm $ echo $? 102 $ nix-build -E 'with import {}; runCommand "foo" {} "date +%s > $out"' --check warning: rewriting hashes in '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo'; cross fingers error: derivation '/nix/store/mggc8dz13ackb49qca6m23zq4fpq132q-foo.drv' may not be deterministic: output '/nix/store/g3k47g0399fvjmbm0p0mnad74k4w8vkz-foo' differs $ echo $? 104 --- src/libstore/build.cc | 16 +++++++++++++--- src/libstore/download.cc | 3 ++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5b38bcf3c..c076ea8b0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -266,6 +266,12 @@ public: /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr hook; @@ -3213,6 +3219,7 @@ void DerivationGoal::registerOutputs() /* Throw an error after registering the path as valid. */ + worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", dest, h.to_string(), h2.to_string())); @@ -3255,6 +3262,7 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); @@ -3266,10 +3274,10 @@ void DerivationGoal::registerOutputs() buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") % drvPath % path); } @@ -4101,6 +4109,8 @@ Worker::Worker(LocalStore & store) lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } @@ -4461,7 +4471,7 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1))); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 0c5a73ea3..7a2af237e 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -855,10 +855,11 @@ CachedDownloadResult Downloader::downloadCached( } if (expectedStorePath != "" && storePath != expectedStorePath) { + unsigned int statusCode = 102; Hash gotHash = request.unpack ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first : hashFile(request.expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, request.expectedHash.to_string(), gotHash.to_string()); } From cbf84bcce7d3fddb353d3aa0f012f5cbdbb77629 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 11 May 2019 23:14:19 +0200 Subject: [PATCH 0360/2548] build: use binary mask for build status flags If multiple builds with fail with different errors it will be reflected in the status code. eg. 103 => timeout + hash mismatch 105 => timeout + check mismatch 106 => hash mismatch + check mismatch 107 => timeout + hash mismatch + check mismatch --- src/libstore/build.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c076ea8b0..abfae3c7c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4471,7 +4471,15 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return checkMismatch ? 104 : (hashMismatch ? 102 : (timedOut ? 101 : (permanentFailure ? 100 : 1))); + unsigned int mask = 0; + if (timedOut) + mask |= 1; + if (hashMismatch) + mask |= 2; + if (checkMismatch) + mask |= 4; + + return mask ? 100 + mask : 1; } From 99ee3755dd12e18f7085b8319a27798b0e5b7de9 Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 11 May 2019 23:32:53 +0200 Subject: [PATCH 0361/2548] build: add tests for --check status codes --- tests/check.nix | 5 +++++ tests/check.sh | 23 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/check.nix b/tests/check.nix index 08aac2fb0..56c82e565 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -10,6 +10,11 @@ with import ./config.nix; ''; }; + hashmismatch = import { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + fetchurl = import { url = "file://" + toString ./lang/eval-okay-xml.exp.xml; sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; diff --git a/tests/check.sh b/tests/check.sh index b05e40ffb..aa1e5cd26 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -6,14 +6,16 @@ nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link -(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +[ "$status" = "104" ] grep 'may not be deterministic' $TEST_ROOT/log clearStore nix-build dependencies.nix --no-out-link --repeat 3 -(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') @@ -23,10 +25,23 @@ echo foo > $path chmod -w $path nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' - # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' - [[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "106" ] From 1ac399dd115d5c86629389e0cdfefa0d654fc90a Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Mon, 13 May 2019 21:03:36 +0200 Subject: [PATCH 0362/2548] nix-store: document exit codes --- doc/manual/command-ref/nix-store.xml | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee..bf12d06f1 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -215,6 +215,41 @@ printed.) +Special exit codes: + + + + 100 + Generic build failure, the builder process + returned with a non-zero exit code. + + + 101 + Build timeout, the build was aborted because it + did not complete within the specified timeout. + + + + 102 + Hash mismatch, the build output was rejected + because it does not match the specified outputHash. + + + + 104 + Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable. + + + + + +With the flag it's possible for +multiple build failures to occur, in this case the 1xx status codes +are or combined. + From a52c331edba6e8c67469f53dda0be2903d18fa8c Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Sat, 15 Jun 2019 15:28:32 +0200 Subject: [PATCH 0363/2548] build: replace 100 offset for build exit codes --- doc/manual/command-ref/nix-store.xml | 11 +++++++++-- src/libstore/build.cc | 24 +++++++++++++++++++----- tests/check.sh | 4 ++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index bf12d06f1..2ecc63db7 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -247,8 +247,15 @@ printed.) With the flag it's possible for -multiple build failures to occur, in this case the 1xx status codes -are or combined. +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index abfae3c7c..350ac4092 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4471,15 +4471,29 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 if (timedOut) - mask |= 1; + mask |= 0x01; // 101 if (hashMismatch) - mask |= 2; - if (checkMismatch) - mask |= 4; + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } - return mask ? 100 + mask : 1; + if (mask) + mask |= 0x60; + return mask ? mask : 1; } diff --git a/tests/check.sh b/tests/check.sh index aa1e5cd26..bc23a6634 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -7,8 +7,8 @@ nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? -[ "$status" = "104" ] grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] clearStore @@ -44,4 +44,4 @@ nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || # Multiple failures with --keep-going nix-build check.nix -A nondeterministic --no-out-link nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? -[ "$status" = "106" ] +[ "$status" = "110" ] From 17d3ec3405eb114f89dc42d065afa887161f6149 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 27 Jun 2019 14:21:22 -0400 Subject: [PATCH 0364/2548] checkStoreName: give more precise/verbose error information $ sudo ./inst/bin/nix-instantiate -E '"${./.git}"' error: The path name '.git' is invalid: it is illegal to start the name with a period. Path names are alphanumeric and can include the symbols +-._?= and must not begin with a period. Note: If '.git' is a source file and you cannot rename it on disk, builtins.path { name = ... } can be used to give it an alternative name. --- src/libstore/store-api.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 28ad7c019..92f01fd2e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -86,18 +86,25 @@ string storePathToHash(const Path & path) void checkStoreName(const string & name) { string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") - throw Error(format("illegal name: '%1%'") % name); + throw Error(baseError % "it is illegal to start the name with a period"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { - throw Error(format("invalid character '%1%' in name '%2%'") - % i % name); + throw Error(baseError % (format("the '%1%' character is invalid") % i)); } } From c8205a3413217ccf8a6a1f7e064b06a5b86c3253 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 2 Jul 2019 09:04:04 -0400 Subject: [PATCH 0365/2548] builtins.fetchGit: document absolute ref support --- doc/manual/expressions/builtins.xml | 15 +++++++++++++++ doc/manual/release-notes/release-notes.xml | 1 + doc/manual/release-notes/rl-2.3.xml | 22 ++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 doc/manual/release-notes/rl-2.3.xml diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 412622714..dcd7aaf24 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -419,6 +419,13 @@ stdenv.mkDerivation { … } This is often a branch or tag name. Defaults to HEAD. + + + By default, the ref value is prefixed + with refs/heads/. As of Nix 2.3.0 + Nix will not prefix refs/heads/ if + ref starts with refs/. + @@ -432,6 +439,14 @@ stdenv.mkDerivation { … } } + + Fetching an arbitrary ref + builtins.fetchGit { + url = "https://gitub.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +} + + Fetching a repository's specific commit on an arbitrary branch diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index e8ff586fa..2655d68e3 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,7 @@ --> + diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000..428213b36 --- /dev/null +++ b/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,22 @@ +
+ +Release 2.3 (????-??-??) + +This release contains the following changes: + + + + + builtins.fetchGit's ref + argument now allows specifying an absolute remote ref. + Nix will automatically prefix ref with + refs/heads only if ref doesn't + already begin with refs/. + + + +
From a3c77c15369703975f7c3e65ebb829580754bc1e Mon Sep 17 00:00:00 2001 From: Daiderd Jordan Date: Thu, 14 Mar 2019 22:39:49 +0100 Subject: [PATCH 0366/2548] nix-store: document --add-fixed --- doc/manual/command-ref/nix-store.xml | 54 ++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee..625598310 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -883,6 +883,60 @@ $ nix-store --add ./foo.c + + +Operation <option>--add-fixed</option> + +Synopsis + + + nix-store + + + algorithm + paths + + + + +Description + +The operation adds the specified paths to +the Nix store. Unlike paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. + + +This operation has the following options: + + + + + + + Use recursive instead of flat hashing mode, used when adding directories + to the store. + + + + + + + + + + +Example + + +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + + + + + From 68bdd83dc88ec55c6c51fa92e84e7d7d408c554a Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 2 Jul 2019 11:18:36 -0400 Subject: [PATCH 0367/2548] timeout: test for error code --- tests/timeout.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/timeout.sh b/tests/timeout.sh index 39ecf0a1a..eea9b5731 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -2,10 +2,14 @@ source common.sh -failed=0 -messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" -if [ $failed -ne 0 ]; then - echo "error: 'nix-store' succeeded; should have timed out" + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi From 96cd3d607374cb4bab27095a62a6f3a832518c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 03:11:52 +0200 Subject: [PATCH 0368/2548] autoconf: Change quotes in description. The unbalanced single-quotes cause many editor syntax highlighters to interpret the rest of the file as a string literal, making it easier to make syntax mistakes in absence of proper highlighting. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f5b1614f1..398bac047 100644 --- a/configure.ac +++ b/configure.ac @@ -42,7 +42,7 @@ esac AC_MSG_RESULT($system) AC_SUBST(system) -AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. From 00a450026f71306bef6acc150af4aa5ffe75801c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 03:12:58 +0200 Subject: [PATCH 0369/2548] autoconf: Detect boost, require version, set CXXFLAGS. This turns previous compiler errors complaining about missing files into proper ./configure time errors telling the user which version of boost is required. --- configure.ac | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/configure.ac b/configure.ac index 398bac047..426d3b674 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,13 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) + + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From 1f97b16b1d9d93e083a4f1436ba002e073cbe379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Mon, 1 Jul 2019 18:31:54 +0200 Subject: [PATCH 0370/2548] autoconf: Work around editline not being found on Ubuntu 16.04. And probably other Linux distributions with long-term support releases. Also update manual stating what version is needed; I checked that 1.14 is the oldest version with which current nix compiles, and added autoconf feature checks for some functions added in that release that nix uses. --- configure.ac | 11 ++++++++++- doc/manual/installation/prerequisites-source.xml | 4 ++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 426d3b674..28be50575 100644 --- a/configure.ac +++ b/configure.ac @@ -171,7 +171,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) # Look for editline, a required dependency. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index e87d0de21..cbaa4d525 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -62,6 +62,10 @@ 1.66.0 or higher. It can be obtained from the official web site . + The editline library of version + 1.14.0 or higher. It can be obtained from the its repository + . + The xmllint and xsltproc programs to build this manual and the man-pages. These are part of the libxml2 and From 57daa860e8ed8432937aeecdcf6b9e952b0481b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Mon, 1 Jul 2019 19:04:03 +0200 Subject: [PATCH 0371/2548] autoconf: Fix C++17 detection not working on Ubuntu 16.04. And probably many other distributions. Until now, ./configure would fail silently printing a warning ./configure: line 4621: AX_CXX_COMPILE_STDCXX_17: command not found and then continuing, later failing with a C++ #error saying that some C++11 feature isn't supported (it didn't even get to the C++17 features). This is because older distributions don't come with the `AX_CXX_COMPILE_STDCXX_17` m4 macro. This commit vendors that macro accordingly. Now ./configure complains correctly: configure: error: *** A compiler with support for C++17 language features is required. On Ubuntu 16.04, ./configure completes if a newer compiler is used, e.g. with gcc-7 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test using: ./bootstrap.sh ./configure CXX=g++-7 --disable-doc-gen --with-boost=$(nix-build --no-link '' -A boost.dev) --- configure.ac | 3 +- .../installation/prerequisites-source.xml | 2 +- m4/ax_cxx_compile_stdcxx.m4 | 951 ++++++++++++++++++ m4/ax_cxx_compile_stdcxx_17.m4 | 35 + 4 files changed, 989 insertions(+), 2 deletions(-) create mode 100644 m4/ax_cxx_compile_stdcxx.m4 create mode 100644 m4/ax_cxx_compile_stdcxx_17.m4 diff --git a/configure.ac b/configure.ac index 28be50575..c12f9d5ff 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,5 @@ AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -62,7 +63,7 @@ CXXFLAGS= AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_17 +AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) AC_CHECK_TOOL([AR], [ar]) diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index cbaa4d525..e7bdcf966 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -13,7 +13,7 @@ Bash Shell. The ./configure script relies on bashisms, so Bash is required. - A version of GCC or Clang that supports C++14. + A version of GCC or Clang that supports C++17. pkg-config to locate dependencies. If your distribution does not provide it, you can get diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 000000000..43087b2e6 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 000000000..a68341717 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016 Krzesimir Nowak +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) From fe068eca00d6a2dd2ea9ee08924dc3898c37fb6c Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 24 Jan 2017 22:42:15 +0000 Subject: [PATCH 0372/2548] mk: add support for passing LDFLAGS to libs and bins autotools-based systems usually allow user to append own LDFLAGS like LDFLAGS="-Wl,-O1 -Wl,--as-needed -Wl,--hash-style=gnu" at ./configure stage This change plumbs LDFLAGS through similar to existing CXXFLAGS variable. Signed-off-by: Sergei Trofimovich --- Makefile.config.in | 1 + mk/libraries.mk | 4 ++-- mk/programs.mk | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646..eabf1a426 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,6 +5,7 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_READLINE = @HAVE_READLINE@ diff --git a/mk/libraries.mk b/mk/libraries.mk index 3953446cb..307e29b9d 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -91,7 +91,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifneq ($(OS), Darwin) $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -105,7 +105,7 @@ define build-library $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifneq ($(OS), Darwin) diff --git a/mk/programs.mk b/mk/programs.mk index 2fbda12bd..d93df4468 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) @@ -46,7 +46,7 @@ define build-program _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) else From 20129bd83d57316cc0b69bec7abeec43011c56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 22:28:30 +0200 Subject: [PATCH 0373/2548] autoconf: Fix AC_STRUCT_DIRENT_D_TYPE being used before AC_PROG_CC. That was incorrect, because checking the dirent type already requires a working compiler. It had the effect that setting e.g. `: ${CFLAGS=""}` before `AC_PROG_CC` as per `AC_PROG_CC`'s documentation would have no effect, because `AC_STRUCT_DIRENT_D_TYPE` would automatically set CFLASGS. (In a followup commit `: ${CFLAGS=""}` will be used, so it's important to get this working first.) --- configure.ac | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index c12f9d5ff..3a486fab8 100644 --- a/configure.ac +++ b/configure.ac @@ -50,14 +50,6 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then - # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi - - CFLAGS= CXXFLAGS= AC_PROG_CC @@ -71,6 +63,14 @@ AC_CHECK_TOOL([AR], [ar]) AC_SYS_LARGEFILE +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) From 717e821b99797845e1bef47d862f8cb0fb69cfc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Tue, 2 Jul 2019 22:30:15 +0200 Subject: [PATCH 0374/2548] autoconf: Allow overriding CFLAGS/CXXFLAGS from outside. As is normal for autoconf-based projects. For example, it is a common use case to do ./configure CXXFLAGS=-O0 This did not work for nix until now, because the `CXXFLAGS=` declaration would unconditionally erase what the user had specified. The custom `OPTIMIZE` flag is removed, but the default `-O3` is retained; autoconf would default to `-g -O2` by default otherwise as documented on: https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C-Compiler.html https://www.gnu.org/software/autoconf/manual/autoconf-2.60/html_node/C_002b_002b-Compiler.html --- Makefile | 7 ------- configure.ac | 6 ++++-- perl/Makefile | 7 ------- perl/configure.ac | 6 ++++-- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 45a3338ed..9ac82fda6 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/configure.ac b/configure.ac index 3a486fab8..571bb54d8 100644 --- a/configure.ac +++ b/configure.ac @@ -50,8 +50,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AC_PROG_CPP diff --git a/perl/Makefile b/perl/Makefile index 284c75022..f36f5d0e9 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index 966700695..e8e3610a8 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -2,8 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 From b49c3a9db516a87d38476966eff9994c13c5acf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 01:34:21 +0200 Subject: [PATCH 0375/2548] Makefile.config.in: Remove HAVE_READLINE. It was forgotten to be removed with commit c5f23f10a84f568874321c04984b1a14d2dce978 and so it until now stayed unsubstituted as `HAVE_READLINE = @HAVE_READLINE@` in Makefile.config. --- Makefile.config.in | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.config.in b/Makefile.config.in index eabf1a426..936a4f403 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,7 +8,6 @@ CXXFLAGS = @CXXFLAGS@ LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ -HAVE_READLINE = @HAVE_READLINE@ HAVE_SECCOMP = @HAVE_SECCOMP@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ From d203c554faa00fec55377d6640c8fb335a611a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 01:46:07 +0200 Subject: [PATCH 0376/2548] Fix C++ compatibility with older editline versions. For example, Ubuntu 16.04 and many similar long-term-support distros have older versions. --- src/nix/repl.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d8f812149..f857b2e89 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -9,7 +9,14 @@ #include #include #else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { #include +} #endif #include "shared.hh" From a96006d97fc87c5073f9a39db841625bdb7c401c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 02:47:27 +0200 Subject: [PATCH 0377/2548] Get BOOST_LDFLAGS from autoconf, fix Ubuntu 16.04 build. Our use of boost::coroutine2 depends on -lboost_context, which in turn depends on `-lboost_thread`, which in turn depends on `-lboost_system`. I suspect that this builds on nix only because of low-level hacks like NIX_LDFLAGS. This commit passes the proper linker flags, thus fixing bootstrap builds on non-nix distributions like Ubuntu 16.04. With these changes, I can build Nix on Ubuntu 16.04 using: ./bootstrap.sh ./configure --prefix=$HOME/editline-prefix \ --disable-doc-gen \ CXX=g++-7 \ --with-boost=$HOME/boost-prefix \ EDITLINE_CFLAGS=-I$HOME/editline-prefix/include \ EDITLINE_LIBS=-leditline \ LDFLAGS=-L$HOME/editline-prefix/lib make where * g++-7 comes from gcc-7 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test, * editline 1.14 from https://github.com/troglobit/editline/releases/tag/1.14.0 was installed into `$HOME/editline-prefix` (because Ubuntu 16.04's `editline` is too old to have the function nix uses), * boost 1.66 from https://www.boost.org/doc/libs/1_66_0/more/getting_started/unix-variants.html was installed into $HOME/boost-prefix (because Ubuntu 16.04 only has 1.58) --- Makefile.config.in | 1 + configure.ac | 3 +++ src/libutil/local.mk | 2 +- src/nix/local.mk | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 936a4f403..7e3b35b98 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -9,6 +9,7 @@ LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ HAVE_SECCOMP = @HAVE_SECCOMP@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ diff --git a/configure.ac b/configure.ac index 571bb54d8..a52830b38 100644 --- a/configure.ac +++ b/configure.ac @@ -153,6 +153,9 @@ AC_SUBST(storedir) # and CPPFLAGS are not passed to the C++ compiler automatically. # Thus we append the returned CPPFLAGS to the CXXFLAGS here. AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" # Look for OpenSSL, a required dependency. diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 3ccc23fd5..e41a67d1f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,4 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d56..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -17,7 +17,7 @@ nix_SOURCES := \ nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ From c3db9e6f8fd06d691be04cdd95a6bb21a400481d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 03:34:17 +0200 Subject: [PATCH 0378/2548] autoconf: Check if --nonet works. Fixes #967 #506. Also give a helpful error message on what package the user likely has to install to make it work. --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index a52830b38..f510f8a97 100644 --- a/configure.ac +++ b/configure.ac @@ -258,6 +258,15 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) +# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) +if test "$doc_generate" = yes; then + AC_MSG_CHECKING([if docbook can run without networking]) + if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then + AC_MSG_RESULT(yes) + else + AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?]) + fi +fi # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) From cd8bc06e8786018ddb16cea4cb10971b63d0efd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 04:20:37 +0200 Subject: [PATCH 0379/2548] autoconf: Add comment on use of `false`. This is to avoid confusion as in commit a0d29040f79b365598fe75d01f72d29ab538206b. --- configure.ac | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index f510f8a97..1e3a8091c 100644 --- a/configure.ac +++ b/configure.ac @@ -119,10 +119,12 @@ if test -z "$$1"; then fi ]) +# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program +# is not required in all cases. (e.g. not when building from a release tarball). +# The use of `false` isn't ideal, because when used somewhere in the build it +# will silently fail without error message; this should be improved. NEED_PROG(bash, bash) NEED_PROG(patch, patch) -AC_PATH_PROG(xmllint, xmllint, false) -AC_PATH_PROG(xsltproc, xsltproc, false) AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) NEED_PROG(sed, sed) @@ -258,8 +260,12 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -# Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) if test "$doc_generate" = yes; then + # Programs required to build the manual + NEED_PROG(xmllint, xmllint) + NEED_PROG(xsltproc, xsltproc) + + # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) AC_MSG_CHECKING([if docbook can run without networking]) if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then AC_MSG_RESULT(yes) From 82b7f0e840983879a510245903ff7c917276f65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Wed, 3 Jul 2019 04:28:06 +0200 Subject: [PATCH 0380/2548] autoconf: Implement release tarball detection. Fixes #257. This should finally allow us to address all cases of build errors due to differences between release tarballs and building from git. See also https://github.com/NixOS/nix/issues/506#issuecomment-507312587 --- configure.ac | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index 1e3a8091c..8bc14d476 100644 --- a/configure.ac +++ b/configure.ac @@ -5,6 +5,19 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED +# Detect building from release tarballs (we need less dependencies then) +# by checking for the `.dist-files` file present only in release tarballs +# (created in release.nix). +# For example, we don't need to build the manual or run bison/flex +# parser generators for release tarballs, as those have them pre-generated. +AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)]) +if test -f .dist-files; then + AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs) + building_from_release_tarball=yes; +else + AC_MSG_RESULT(no) +fi + # Construct a Nix system name (like "i686-linux"). AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -119,14 +132,13 @@ if test -z "$$1"; then fi ]) -# Note: Usage of `AC_PATH_PROG(..., ..., false)` indicates that this program -# is not required in all cases. (e.g. not when building from a release tarball). -# The use of `false` isn't ideal, because when used somewhere in the build it -# will silently fail without error message; this should be improved. NEED_PROG(bash, bash) NEED_PROG(patch, patch) -AC_PATH_PROG(flex, flex, false) -AC_PATH_PROG(bison, bison, false) +if test "x$building_from_release_tarball" != x"yes"; then + # Parser generators are pre-built in release tarballs. + NEED_PROG(flex, flex) + NEED_PROG(bison, bison) +fi NEED_PROG(sed, sed) NEED_PROG(tar, tar) NEED_PROG(bzip2, bzip2) @@ -260,7 +272,8 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -if test "$doc_generate" = yes; then +# The manual is pre-generated in release tarballs. +if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then # Programs required to build the manual NEED_PROG(xmllint, xmllint) NEED_PROG(xsltproc, xsltproc) From e486d8d40e626a20e06d792db8cc5ac5aba9a5b4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Jul 2019 00:33:51 +0200 Subject: [PATCH 0381/2548] Revert 82b7f0e840983879a510245903ff7c917276f65d, cd8bc06e8786018ddb16cea4cb10971b63d0efd2, c3db9e6f8fd06d691be04cdd95a6bb21a400481d This breaks the tarball job: https://hydra.nixos.org/build/95714570 --- configure.ac | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/configure.ac b/configure.ac index 8bc14d476..a52830b38 100644 --- a/configure.ac +++ b/configure.ac @@ -5,19 +5,6 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Detect building from release tarballs (we need less dependencies then) -# by checking for the `.dist-files` file present only in release tarballs -# (created in release.nix). -# For example, we don't need to build the manual or run bison/flex -# parser generators for release tarballs, as those have them pre-generated. -AC_MSG_CHECKING([whether we are building from a release tarball (.dist-files existence)]) -if test -f .dist-files; then - AC_MSG_RESULT(yes; will not require tools that are not needed for building from release tarballs) - building_from_release_tarball=yes; -else - AC_MSG_RESULT(no) -fi - # Construct a Nix system name (like "i686-linux"). AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -134,11 +121,10 @@ fi NEED_PROG(bash, bash) NEED_PROG(patch, patch) -if test "x$building_from_release_tarball" != x"yes"; then - # Parser generators are pre-built in release tarballs. - NEED_PROG(flex, flex) - NEED_PROG(bison, bison) -fi +AC_PATH_PROG(xmllint, xmllint, false) +AC_PATH_PROG(xsltproc, xsltproc, false) +AC_PATH_PROG(flex, flex, false) +AC_PATH_PROG(bison, bison, false) NEED_PROG(sed, sed) NEED_PROG(tar, tar) NEED_PROG(bzip2, bzip2) @@ -272,20 +258,6 @@ AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) -# The manual is pre-generated in release tarballs. -if test "$doc_generate" = yes && test "x$building_from_release_tarball" != x"yes"; then - # Programs required to build the manual - NEED_PROG(xmllint, xmllint) - NEED_PROG(xsltproc, xsltproc) - - # Check if docbook works without Internet (if not, likely packages providing the .xsl files are not installed) - AC_MSG_CHECKING([if docbook can run without networking]) - if "$xsltproc" --nonet --novalid http://docbook.sourceforge.net/release/xsl-ns/current/profiling/profile.xsl >&AS_MESSAGE_LOG_FD 2>&1; then - AC_MSG_RESULT(yes) - else - AC_MSG_FAILURE([Building the manual requires 'xsltproc --nonet' to work. It did not. Perhaps you lack the necessary xsl files (e.g. package docbook-xsl-ns on Debian)?]) - fi -fi # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) From 648bdf153d2e4d3c6687d8e1780b0dfd0aa98cda Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 21 May 2019 22:29:23 -0400 Subject: [PATCH 0382/2548] tarball-ttl: document MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Incorporates text from Niklas Hambüchen in #2978 Closes #1115 --- doc/manual/command-ref/conf-file.xml | 25 +++++++++++++++++++++++++ doc/manual/expressions/builtins.xml | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 24fbf28cf..09aad2e05 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -864,6 +864,31 @@ requiredSystemFeatures = [ "kvm" ]; + tarball-ttl + + + Default: 3600 seconds. + + The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + + + Setting the TTL to 0 forces Nix to always + check if the tarball is up to date. + + Nix caches tarballs in + $XDG_CACHE_HOME/nix/tarballs. + + Files fetched via NIX_PATH, + fetchGit, fetchMercurial, + fetchTarball, and fetchurl + respect this TTL. + + + timeout diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index a87639a07..deffcb594 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -347,7 +347,7 @@ stdenv.mkDerivation { … } You can change the cache timeout either on the command line with or in the Nix configuration file with this option: - tarball-ttl number of seconds to cache. + number of seconds to cache. Note that when obtaining the hash with nix-prefetch-url @@ -498,7 +498,7 @@ stdenv.mkDerivation { … } fetch the latest version of a remote branch. Nix will refetch the branch in accordance to - . + . This behavior is disabled in Pure evaluation mode. builtins.fetchGit { From b0c220c02ec584af282b9c7f493e4a4d2e429f8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 10:27:33 +0200 Subject: [PATCH 0383/2548] Check for epochs < 201906 --- src/libexpr/flake/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 302549a3c..4f59c61bd 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -227,6 +227,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (auto epoch = vInfo.attrs->get(sEpoch)) { flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch < 201906) + throw Error("flake '%s' has illegal epoch %d", flakeRef, flake.epoch); if (flake.epoch > 201906) throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); } else From aa739e783993864aa6e0c8a4820e6b59f4626d92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 10:29:41 +0200 Subject: [PATCH 0384/2548] nix copy: Rename --substitute to --substitute-on-destination '--substitute' was being shadowed by the regular '--substitute' (the short-hand for '--option substitute true'). Fixes #2983. --- src/nix/copy.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 96bd453d8..12a9f9cd3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -36,7 +36,7 @@ struct CmdCopy : StorePathsCommand .set(&checkSigs, NoCheckSigs); mkFlag() - .longName("substitute") + .longName("substitute-on-destination") .shortName('s') .description("whether to try substitutes on the destination store (only supported by SSH)") .set(&substitute, Substitute); From 03f09e1d18c39706b917964f14f9e40513e4aaea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 19:46:15 +0200 Subject: [PATCH 0385/2548] Revert "Fix 'error 9 while decompressing xz file'" This reverts commit 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91. --- src/libstore/binary-cache-store.cc | 18 +++--- src/libstore/download.cc | 67 ++++++++++++++++---- src/libstore/download.hh | 11 +--- src/libstore/http-binary-cache-store.cc | 55 +++++----------- src/libstore/store-api.cc | 84 ++++++++++++------------- src/libutil/retry.hh | 38 ----------- src/libutil/types.hh | 2 - 7 files changed, 119 insertions(+), 156 deletions(-) delete mode 100644 src/libutil/retry.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8b736056e..4527ee6ba 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,8 +10,6 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" -#include "retry.hh" -#include "download.hh" #include @@ -81,15 +79,13 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) std::shared_ptr BinaryCacheStore::getFile(const std::string & path) { - return retry>(downloadSettings.tries, [&]() -> std::shared_ptr { - StringSink sink; - try { - getFile(path, sink); - } catch (NoSuchBinaryCacheFile &) { - return nullptr; - } - return sink.s; - }); + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; } Path BinaryCacheStore::narInfoFileFor(const Path & storePath) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 7a2af237e..2267b5d35 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -8,7 +8,6 @@ #include "compression.hh" #include "pathlocks.hh" #include "finally.hh" -#include "retry.hh" #ifdef ENABLE_S3 #include @@ -20,9 +19,11 @@ #include #include +#include #include #include #include +#include #include using namespace std::string_literals; @@ -45,6 +46,9 @@ struct CurlDownloader : public Downloader { CURLM * curlm = 0; + std::random_device rd; + std::mt19937 mt19937; + struct DownloadItem : public std::enable_shared_from_this { CurlDownloader & downloader; @@ -57,6 +61,12 @@ struct CurlDownloader : public Downloader bool active = false; // whether the handle has been added to the multi object std::string status; + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + struct curl_slist * requestHeaders = 0; std::string encoding; @@ -377,7 +387,9 @@ struct CurlDownloader : public Downloader } } - fail( + attempt++; + + auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 @@ -388,15 +400,31 @@ struct CurlDownloader : public Downloader ) : DownloadError(err, fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code))); + request.verb(), request.uri, curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. */ + if (err == Transient && attempt < request.tries) { + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + downloader.enqueueItem(shared_from_this()); + } + else + fail(exc); } } }; struct State { + struct EmbargoComparator { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + return i1->embargo > i2->embargo; + } + }; bool quit = false; - std::vector> incoming; + std::priority_queue, std::vector>, EmbargoComparator> incoming; }; Sync state_; @@ -409,6 +437,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; CurlDownloader() + : mt19937(rd()) { static std::once_flag globalInit; std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); @@ -502,7 +531,9 @@ struct CurlDownloader : public Downloader nextWakeup = std::chrono::steady_clock::time_point(); - /* Add new curl requests from the incoming requests queue. */ + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ if (extraFDs[0].revents & CURL_WAIT_POLLIN) { char buf[1024]; auto res = read(extraFDs[0].fd, buf, sizeof(buf)); @@ -511,9 +542,22 @@ struct CurlDownloader : public Downloader } std::vector> incoming; + auto now = std::chrono::steady_clock::now(); + { auto state(state_.lock()); - std::swap(state->incoming, incoming); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() + || item->embargo < nextWakeup) + nextWakeup = item->embargo; + break; + } + } quit = state->quit; } @@ -540,7 +584,7 @@ struct CurlDownloader : public Downloader { auto state(state_.lock()); - state->incoming.clear(); + while (!state->incoming.empty()) state->incoming.pop(); state->quit = true; } } @@ -556,7 +600,7 @@ struct CurlDownloader : public Downloader auto state(state_.lock()); if (state->quit) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); - state->incoming.push_back(item); + state->incoming.push(item); } writeFull(wakeupPipe.writeSide.get(), " "); } @@ -639,9 +683,7 @@ std::future Downloader::enqueueDownload(const DownloadRequest & DownloadResult Downloader::download(const DownloadRequest & request) { - return retry(request.tries, [&]() { - return enqueueDownload(request).get(); - }); + return enqueueDownload(request).get(); } void Downloader::download(DownloadRequest && request, Sink & sink) @@ -827,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - warn("%s; using cached result", e.msg()); + printError(format("warning: %1%; using cached result") % e.msg()); result.etag = expectedETag; } } @@ -878,4 +920,5 @@ bool isUri(const string & s) return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } + } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 9e965b506..3b7fff3ba 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -96,13 +96,11 @@ struct Downloader std::future enqueueDownload(const DownloadRequest & request); - /* Synchronously download a file. The request will be retried in - case of transient failures. */ + /* Synchronously download a file. */ DownloadResult download(const DownloadRequest & request); /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. The request will not be - retried in case of transient failures. */ + invoked on the thread of the caller. */ void download(DownloadRequest && request, Sink & sink); /* Check if the specified file is already in ~/.cache/nix/tarballs @@ -128,11 +126,6 @@ public: DownloadError(Downloader::Error error, const FormatOrString & fs) : Error(fs), error(error) { } - - bool isTransient() override - { - return error == Downloader::Error::Transient; - } }; bool isUri(const string & s); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5633b4355..3d17a8966 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -2,7 +2,6 @@ #include "download.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" -#include "retry.hh" namespace nix { @@ -114,6 +113,7 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); + request.tries = 8; return request; } @@ -136,46 +136,21 @@ protected: { checkEnabled(); - struct State - { - DownloadRequest request; - std::function tryDownload; - unsigned int attempt = 0; - State(DownloadRequest && request) : request(request) {} - }; + auto request(makeRequest(path)); - auto state = std::make_shared(makeRequest(path)); - - state->tryDownload = [callback, state, this]() { - getDownloader()->enqueueDownload(state->request, - {[callback, state, this](std::future result) { - try { - callback(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); - ++state->attempt; - if (state->attempt < state->request.tries && e.isTransient()) { - auto ms = retrySleepTime(state->attempt); - warn("%s; retrying in %d ms", e.what(), ms); - /* We can't sleep here because that would - block the download thread. So use a - separate thread for sleeping. */ - std::thread([state, ms]() { - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); - state->tryDownload(); - }).detach(); - } else { - maybeDisable(); - callback.rethrow(); - } - } catch (...) { - callback.rethrow(); - } - }}); - }; - - state->tryDownload(); + getDownloader()->enqueueDownload(request, + {[callback, this](std::future result) { + try { + callback(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return callback(std::shared_ptr()); + maybeDisable(); + callback.rethrow(); + } catch (...) { + callback.rethrow(); + } + }}); } }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 92f01fd2e..f5608d384 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,11 +6,10 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" -#include "retry.hh" -#include "download.hh" #include + namespace nix { @@ -580,57 +579,54 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) void copyStorePath(ref srcStore, ref dstStore, const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { - retry(downloadSettings.tries, [&]() { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); - Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); - PushActivity pact(act.id); + auto info = srcStore->queryPathInfo(storePath); - auto info = srcStore->queryPathInfo(storePath); + uint64_t total = 0; - uint64_t total = 0; + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; - if (!info->narHash) { - StringSink sink; - srcStore->narFromPath({storePath}, sink); - auto info2 = make_ref(*info); - info2->narHash = hashString(htSHA256, *sink.s); - if (!info->narSize) info2->narSize = sink.s->size(); - if (info->ultimate) info2->ultimate = false; - info = info2; + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } - StringSource source(*sink.s); - dstStore->addToStore(*info, source, repair, checkSigs); - return; - } + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } - if (info->ultimate) { - auto info2 = make_ref(*info); - info2->ultimate = false; - info = info2; - } - - auto source = sinkToSource([&](Sink & sink) { - LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { - sink(data, len); - total += len; - act.progress(total, info->narSize); - }); - srcStore->narFromPath({storePath}, wrapperSink); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); }); - - dstStore->addToStore(*info, *source, repair, checkSigs); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); + + dstStore->addToStore(*info, *source, repair, checkSigs); } diff --git a/src/libutil/retry.hh b/src/libutil/retry.hh deleted file mode 100644 index b45cb37f7..000000000 --- a/src/libutil/retry.hh +++ /dev/null @@ -1,38 +0,0 @@ -#pragma once - -#include "logging.hh" - -#include -#include -#include -#include - -namespace nix { - -inline unsigned int retrySleepTime(unsigned int attempt) -{ - std::random_device rd; - std::mt19937 mt19937; - return 250.0 * std::pow(2.0f, - attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(mt19937)); -} - -template -C retry(unsigned int attempts, std::function && f) -{ - unsigned int attempt = 0; - while (true) { - try { - return f(); - } catch (BaseError & e) { - ++attempt; - if (attempt >= attempts || !e.isTransient()) - throw; - auto ms = retrySleepTime(attempt); - warn("%s; retrying in %d ms", e.what(), ms); - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); - } - } -} - -} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 88e3243f4..92bf469b5 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -109,8 +109,6 @@ public: const string & msg() const { return err; } const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); - - virtual bool isTransient() { return false; } }; #define MakeError(newClass, superClass) \ From f76b2a7fdd7dabad7891400248f5cc778182ebf3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 22:27:50 +0200 Subject: [PATCH 0386/2548] Downloader: Use warn() --- src/libstore/download.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 2267b5d35..195eccc2a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -406,7 +406,7 @@ struct CurlDownloader : public Downloader download after a while. */ if (err == Transient && attempt < request.tries) { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } @@ -579,7 +579,7 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError(format("unexpected error in download thread: %s") % e.what()); + printError("unexpected error in download thread: %s", e.what()); } { @@ -869,7 +869,7 @@ CachedDownloadResult Downloader::downloadCached( writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("warning: %s; using cached result", e.msg()); result.etag = expectedETag; } } From 00f6fafad61db4537268e1ffa636fd0d2ae86059 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 23:05:04 +0200 Subject: [PATCH 0387/2548] HttpBinaryCacheStore: Use default number of retries for NARs --- src/libstore/http-binary-cache-store.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3d17a8966..df2fb9332 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -113,7 +113,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } From 53247d6b116905e7233b1efd6c14845e20d27442 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Jul 2019 23:12:17 +0200 Subject: [PATCH 0388/2548] Resume NAR downloads This is a much simpler fix to the 'error 9 while decompressing xz file' problem than 78fa47a7f08a4cb6ee7061bf0bd86a40e1d6dc91. We just do a ranged HTTP request starting after the data that we previously wrote into the sink. Fixes #2952, #379. --- src/libstore/download.cc | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 195eccc2a..91087eebc 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -71,6 +71,10 @@ struct CurlDownloader : public Downloader std::string encoding; + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, Callback callback) @@ -81,9 +85,10 @@ struct CurlDownloader : public Downloader {request.uri}, request.parentAct) , callback(callback) , finalSink([this](const unsigned char * data, size_t len) { - if (this->request.dataCallback) + if (this->request.dataCallback) { + writtenToSink += len; this->request.dataCallback((char *) data, len); - else + } else this->result.data->append((char *) data, len); }) { @@ -161,6 +166,7 @@ struct CurlDownloader : public Downloader status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared(); result.bodySize = 0; + acceptRanges = false; encoding = ""; } else { auto i = line.find(':'); @@ -178,7 +184,9 @@ struct CurlDownloader : public Downloader return 0; } } else if (name == "content-encoding") - encoding = trim(string(line, i + 1));; + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; } } return realSize; @@ -296,6 +304,9 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + result.data = std::make_shared(); result.bodySize = 0; } @@ -330,7 +341,7 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; done = true; @@ -403,10 +414,20 @@ struct CurlDownloader : public Downloader request.verb(), request.uri, curl_easy_strerror(code), code)); /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - warn("%s; retrying in %d ms", exc.what(), ms); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } From ad42a784690449873fccb20192bd2150da81c56d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jul 2019 13:54:53 +0200 Subject: [PATCH 0389/2548] Rename 'epoch' -> 'edition' --- doc/flakes/design.md | 10 +++++----- src/libexpr/flake/flake.cc | 24 +++++++++++++++--------- src/libexpr/flake/flake.hh | 2 +- src/nix/flake.cc | 4 ++-- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/doc/flakes/design.md b/doc/flakes/design.md index c9520bcbf..97bd12ce3 100644 --- a/doc/flakes/design.md +++ b/doc/flakes/design.md @@ -65,7 +65,7 @@ Upcoming but not yet implemented: NixOS configuration can be reproduced unambiguously from the top-level flake. * Nix code can query flake metadata such as `commitHash` (the Git revision) or - `epoch` (the date of the last commit). This is useful for NixOS to compute + `edition` (the date of the last commit). This is useful for NixOS to compute the NixOS version string (which will be the revision of the top-level configuration flake, uniquely identifying the configuration). @@ -85,9 +85,9 @@ repository that provides a single package and a single NixOS module. # The flake identifier. name = "dwarffs"; - # The epoch may be used in the future to determine how Nix + # The edition may be used in the future to determine how Nix # expressions inside this flake are to be parsed. - epoch = 201906; + edition = 201906; # Some other metadata. description = "A filesystem that fetches DWARF debug info from the Internet on demand"; @@ -162,7 +162,7 @@ Similarly, a minimal `flake.nix` for Nixpkgs: { name = "nixpkgs"; - epoch = 201906; + edition = 201906; description = "A collection of packages for the Nix package manager"; @@ -449,7 +449,7 @@ flakes in (local) Git repositories. { name = "my-system"; - epoch = 201906; + edition = 201906; inputs = [ "nixpkgs/nixos-18.09" diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 4f59c61bd..8b9525680 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -223,16 +223,21 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) state.forceAttrs(vInfo); - auto sEpoch = state.symbols.create("epoch"); + auto sEdition = state.symbols.create("edition"); + auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon - if (auto epoch = vInfo.attrs->get(sEpoch)) { - flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); - if (flake.epoch < 201906) - throw Error("flake '%s' has illegal epoch %d", flakeRef, flake.epoch); - if (flake.epoch > 201906) - throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + auto edition = vInfo.attrs->get(sEdition); + if (!edition) + edition = vInfo.attrs->get(sEpoch); + + if (edition) { + flake.edition = state.forceInt(*(**edition).value, *(**edition).pos); + if (flake.edition < 201906) + throw Error("flake '%s' has illegal edition %d", flakeRef, flake.edition); + if (flake.edition > 201906) + throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", flakeRef, flake.edition); } else - throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + throw Error("flake '%s' lacks attribute 'edition'", flakeRef); if (auto name = vInfo.attrs->get(state.sName)) flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); @@ -271,7 +276,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); for (auto & attr : *vInfo.attrs) { - if (attr.name != sEpoch && + if (attr.name != sEdition && + attr.name != sEpoch && attr.name != state.sName && attr.name != state.sDescription && attr.name != sInputs && diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index de0feb2c4..01fb421bd 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -67,7 +67,7 @@ struct Flake std::vector inputs; std::map nonFlakeInputs; Value * vOutputs; // FIXME: gc - unsigned int epoch; + unsigned int edition; Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) : originalRef(origRef), sourceInfo(sourceInfo) {}; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 49f7c33c7..aab29b626 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -105,7 +105,7 @@ static void printFlakeInfo(const Flake & flake) { std::cout << fmt("ID: %s\n", flake.id); std::cout << fmt("Description: %s\n", flake.description); - std::cout << fmt("Epoch: %s\n", flake.epoch); + std::cout << fmt("Edition: %s\n", flake.edition); printSourceInfo(flake.sourceInfo); } @@ -114,7 +114,7 @@ static nlohmann::json flakeToJson(const Flake & flake) nlohmann::json j; j["id"] = flake.id; j["description"] = flake.description; - j["epoch"] = flake.epoch; + j["edition"] = flake.edition; sourceInfoToJson(flake.sourceInfo, j); return j; } From 0802e006f27acadc32c43ba02313709dfc51f940 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 11 Jul 2019 17:05:53 +0200 Subject: [PATCH 0390/2548] Use "git add --force --intent-to-add" for flake.lock Fixes The following paths are ignored by one of your .gitignore files: flake.lock --- src/libexpr/flake/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8b9525680..abbb9a3e1 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -409,6 +409,8 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. runProgram("git", true, { "-C", refData->path, "add", + "--force", + "--intent-to-add", (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); } else warn("cannot write lockfile of remote flake '%s'", topRef); From bd62290c23fa15181e8af75c5055e104900f8532 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 12:59:00 +0200 Subject: [PATCH 0391/2548] fetchGit: Warn about dirty trees --- src/libexpr/primops/fetchGit.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 10cd2470a..940cf3022 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -46,6 +46,8 @@ GitInfo exportGit(ref store, std::string uri, /* This is an unclean working tree. So copy all tracked files. */ + warn("Git tree '%s' is dirty", uri); + GitInfo gitInfo; gitInfo.ref = "HEAD"; From b29cec76971ff1949415f76c82fa0ecf699ac264 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 13:29:54 +0200 Subject: [PATCH 0392/2548] Don't write lock files if they have dirty inputs --- src/libexpr/flake/flake.cc | 25 +++++++++++++++---------- src/libexpr/flake/flakeref.hh | 6 ++++++ src/libexpr/flake/lockfile.cc | 11 +++++++++++ src/libexpr/flake/lockfile.hh | 4 ++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index abbb9a3e1..e6cef502c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -405,17 +405,22 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { - lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + if (lockFile.isDirty()) + warn("will not write lock file of flake '%s' because it has a dirty input", topRef); + else { + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); - // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. - runProgram("git", true, { "-C", refData->path, "add", - "--force", - "--intent-to-add", - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, + { "-C", refData->path, "add", + "--force", + "--intent-to-add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } } else - warn("cannot write lockfile of remote flake '%s'", topRef); + warn("cannot write lock file of remote flake '%s'", topRef); } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) - warn("using updated lockfile without writing it to file"); + warn("using updated lock file without writing it to file"); } return ResolvedFlake(std::move(flake), std::move(lockFile)); @@ -624,8 +629,8 @@ const Registries EvalState::getFlakeRegistries() Fingerprint ResolvedFlake::getFingerprint() const { - // FIXME: as an optimization, if the flake contains a lockfile and - // we haven't changed it, then it's sufficient to use + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s", flake.sourceInfo.storePath, lockFile)); diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 52bb82ddb..082dd8c26 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -176,6 +176,12 @@ struct FlakeRef bool isImmutable() const; FlakeRef baseRef() const; + + bool isDirty() const + { + return std::get_if(&data) + && rev == Hash(rev->type); + } }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 97c748c66..15f2e2e8e 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -66,6 +66,17 @@ nlohmann::json FlakeInputs::toJson() const return json; } +bool FlakeInputs::isDirty() const +{ + for (auto & i : flakeInputs) + if (i.second.ref.isDirty() || i.second.isDirty()) return true; + + for (auto & i : nonFlakeInputs) + if (i.second.ref.isDirty()) return true; + + return false; +} + nlohmann::json LockFile::toJson() const { auto json = FlakeInputs::toJson(); diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index b76124190..7077db3cd 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -53,6 +53,10 @@ struct FlakeInputs FlakeInputs(const nlohmann::json & json); nlohmann::json toJson() const; + + /* A lock file is dirty if it contains a dirty flakeref + (i.e. reference to a dirty working tree). */ + bool isDirty() const; }; /* Lock file information about a flake input. */ From b45628a172dd3a3ba863cd231f1446c7cd901cf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 14:37:45 +0200 Subject: [PATCH 0393/2548] Fix flake tests --- tests/flakes.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index d2b168712..406e5b632 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -31,7 +31,7 @@ cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <&1 | grep 'unsupported epoch' +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported edition' # Test whether registry caching works. nix flake list --flake-registry file://$registry | grep -q flake3 @@ -242,7 +244,7 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Fri, 12 Jul 2019 15:32:17 +0200 Subject: [PATCH 0394/2548] nix build: Add '--profile' flag This replaces 'nix-env --set'. For example: $ nix build --profile /nix/var/nix/profiles/system \ ~/Misc/eelco-configurations:nixosConfigurations.vyr.config.system.build.toplevel updates the NixOS system profile from a flake. This could have been a separate command (e.g. 'nix set-profile') but 1) '--profile' is pretty similar to '--out-link'; and 2) '--profile' could be useful for other command (like 'nix dev-shell'). --- src/nix/build.cc | 17 +++++++++++------ src/nix/command.cc | 41 +++++++++++++++++++++++++++++++++++++++++ src/nix/command.hh | 14 ++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index d8ce8cc80..f63150012 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -6,7 +6,7 @@ using namespace nix; -struct CmdBuild : MixDryRun, InstallablesCommand +struct CmdBuild : MixDryRun, MixProfile, InstallablesCommand { Path outLink = "result"; @@ -41,6 +41,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand "To build the build.x86_64-linux attribute from release.nix:", "nix build -f release.nix build.x86_64-linux" }, + Example{ + "To make a profile point at GNU Hello:", + "nix build --profile /tmp/profile nixpkgs:hello" + }, }; } @@ -52,18 +56,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand evalState->addRegistryOverrides(registryOverrides); if (dryRun) return; - for (size_t i = 0; i < buildables.size(); ++i) { - auto & b(buildables[i]); - - if (outLink != "") - for (auto & output : b.outputs) + if (outLink != "") { + for (size_t i = 0; i < buildables.size(); ++i) { + for (auto & output : buildables[i].outputs) if (auto store2 = store.dynamic_pointer_cast()) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); store2->addPermRoot(output.second, absPath(symlink), true); } + } } + + updateProfile(buildables); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index 89fa0cba4..8191cb831 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -1,6 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "derivations.hh" +#include "profiles.hh" namespace nix { @@ -81,4 +82,44 @@ void StorePathCommand::run(ref store) run(store, *storePaths.begin()); } +MixProfile::MixProfile() +{ + mkFlag() + .longName("profile") + .description("profile to update") + .labels({"path"}) + .dest(&profile); +} + +void MixProfile::updateProfile(const Path & storePath) +{ + if (!profile) return; + auto store = getStore().dynamic_pointer_cast(); + if (!store) throw Error("'--profile' is not supported for this Nix store"); + switchLink(*profile, + createGeneration( + ref(store), + *profile, storePath)); +} + +void MixProfile::updateProfile(const Buildables & buildables) +{ + if (!profile) return; + + std::optional result; + + for (auto & buildable : buildables) { + for (auto & output : buildable.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second; + } + } + + if (!result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + + updateProfile(*result); +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index 59c6f8578..d6153e42b 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -219,4 +219,18 @@ PathSet toDerivations(ref store, std::vector> installables, bool useDeriver = false); +struct MixProfile : virtual Args, virtual StoreCommand +{ + std::optional profile; + + MixProfile(); + + /* If 'profile' is set, make it point at 'storePath'. */ + void updateProfile(const Path & storePath); + + /* If 'profile' is set, make it point at the store path produced + by 'buildables'. */ + void updateProfile(const Buildables & buildables); +}; + } From 7ba928116ef1677b7403525df9e8abb49001820e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 16:10:58 +0200 Subject: [PATCH 0395/2548] nix dev-shell: Add --profile flag This is useful to prevent the shell environment from being garbage-collected. --- src/nix/shell.cc | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index f42947b7c..4c7b701dc 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -66,7 +66,7 @@ BuildEnvironment readEnvironment(const Path & path) modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -BuildEnvironment getDerivationEnvironment(ref store, Derivation drv) +Path getDerivationEnvironment(ref store, Derivation drv) { auto builder = baseNameOf(drv.builder); if (builder != "bash") @@ -101,7 +101,7 @@ BuildEnvironment getDerivationEnvironment(ref store, Derivation drv) assert(store->isValidPath(shellOutPath)); - return readEnvironment(shellOutPath); + return shellOutPath; } struct Common : InstallableCommand @@ -175,7 +175,7 @@ struct Common : InstallableCommand } }; -struct CmdDevShell : Common +struct CmdDevShell : Common, MixProfile { std::string description() override { @@ -193,6 +193,10 @@ struct CmdDevShell : Common "To get the build environment of the default package of flake in the current directory:", "nix dev-shell" }, + Example{ + "To store the build environment in a profile:", + "nix dev-shell --profile /tmp/my-shell" + }, }; } @@ -206,7 +210,11 @@ struct CmdDevShell : Common auto & drvPath = *drvs.begin(); - auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + auto shellOutPath = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + updateProfile(shellOutPath); + + auto buildEnvironment = readEnvironment(shellOutPath); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -259,7 +267,9 @@ struct CmdPrintDevEnv : Common auto & drvPath = *drvs.begin(); - auto buildEnvironment = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + auto buildEnvironment = readEnvironment( + getDerivationEnvironment(store, + store->derivationFromPath(drvPath))); stopProgressBar(); From 731bc65ec04900834ca5e5b8e9dae1aa8c2c1027 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 16:16:27 +0200 Subject: [PATCH 0396/2548] Refactor a bit --- src/nix/shell.cc | 47 +++++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 4c7b701dc..442835d38 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -104,7 +104,7 @@ Path getDerivationEnvironment(ref store, Derivation drv) return shellOutPath; } -struct Common : InstallableCommand +struct Common : InstallableCommand, MixProfile { /* std::set keepVars{ @@ -173,9 +173,26 @@ struct Common : InstallableCommand { return {"devShell", "defaultPackage"}; } + + BuildEnvironment getBuildEnvironment(ref store) + { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + auto shellOutPath = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + + updateProfile(shellOutPath); + + return readEnvironment(shellOutPath); + } }; -struct CmdDevShell : Common, MixProfile +struct CmdDevShell : Common { std::string description() override { @@ -202,19 +219,7 @@ struct CmdDevShell : Common, MixProfile void run(ref store) override { - auto drvs = toDerivations(store, {installable}); - - if (drvs.size() != 1) - throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", - installable->what(), drvs.size()); - - auto & drvPath = *drvs.begin(); - - auto shellOutPath = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); - - updateProfile(shellOutPath); - - auto buildEnvironment = readEnvironment(shellOutPath); + auto buildEnvironment = getBuildEnvironment(store); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -259,17 +264,7 @@ struct CmdPrintDevEnv : Common void run(ref store) override { - auto drvs = toDerivations(store, {installable}); - - if (drvs.size() != 1) - throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", - installable->what(), drvs.size()); - - auto & drvPath = *drvs.begin(); - - auto buildEnvironment = readEnvironment( - getDerivationEnvironment(store, - store->derivationFromPath(drvPath))); + auto buildEnvironment = getBuildEnvironment(store); stopProgressBar(); From aa82f8b2d2a2c42f0d713e8404b668cef1a4b108 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Jul 2019 16:28:39 +0200 Subject: [PATCH 0397/2548] nix dev-shell: Make it possible to enter a profile For example: $ nix dev-shell --profile /tmp/my-shell dwarffs (later) $ nix dev-shell /tmp/my-shell --- src/nix/command.hh | 7 +++++++ src/nix/installables.cc | 5 +++++ src/nix/shell.cc | 34 ++++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/nix/command.hh b/src/nix/command.hh index d6153e42b..00c202f20 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -64,6 +64,13 @@ struct Installable { throw Error("argument '%s' cannot be evaluated", what()); } + + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional getStorePath() + { + return {}; + } }; struct EvalCommand : virtual StoreCommand, MixEvalArgs diff --git a/src/nix/installables.cc b/src/nix/installables.cc index d43f86c0c..aa5ef5184 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -102,6 +102,11 @@ struct InstallableStorePath : Installable { return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; } + + std::optional getStorePath() override + { + return storePath; + } }; struct InstallableValue : Installable diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 442835d38..93732f6a3 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -174,17 +174,27 @@ struct Common : InstallableCommand, MixProfile return {"devShell", "defaultPackage"}; } + Path getShellOutPath(ref store) + { + auto path = installable->getStorePath(); + if (path && hasSuffix(*path, "-env")) + return *path; + else { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + return getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + } + } + BuildEnvironment getBuildEnvironment(ref store) { - auto drvs = toDerivations(store, {installable}); - - if (drvs.size() != 1) - throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", - installable->what(), drvs.size()); - - auto & drvPath = *drvs.begin(); - - auto shellOutPath = getDerivationEnvironment(store, store->derivationFromPath(drvPath)); + auto shellOutPath = getShellOutPath(store); updateProfile(shellOutPath); @@ -212,7 +222,11 @@ struct CmdDevShell : Common }, Example{ "To store the build environment in a profile:", - "nix dev-shell --profile /tmp/my-shell" + "nix dev-shell --profile /tmp/my-shell nixpkgs:hello" + }, + Example{ + "To use a build environment previously recorded in a profile:", + "nix dev-shell /tmp/my-shell" }, }; } From 5e0a64229b3d244d46b3b5f9fae964d01958c15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domen=20Ko=C5=BEar?= Date: Thu, 18 Jul 2019 10:57:26 +0200 Subject: [PATCH 0398/2548] Add Open Collective --- .github/FUNDING.yml | 2 +- README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index bbaabf93c..15792e713 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -custom: https://nixos.org/nixos/foundation.html +open_collective: nixos diff --git a/README.md b/README.md index 3173c6c44..48cb1685c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) + Nix, the purely functional package manager ------------------------------------------ From cf6172f05e9446b2b85dd7fa2e73cb93f56620e2 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Thu, 28 Mar 2019 16:04:02 -0400 Subject: [PATCH 0399/2548] docs: document balancing cores and max-jobs --- .../advanced-topics/advanced-topics.xml | 1 + doc/manual/advanced-topics/cores-vs-jobs.xml | 121 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 8 +- 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 doc/manual/advanced-topics/cores-vs-jobs.xml diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..12a826254 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,6 +7,7 @@ Advanced Topics + diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000..eba645faf --- /dev/null +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ + + +Tuning Cores and Jobs + +Nix has two relevant settings with regards to how your CPU cores +will be utilized: and +. This chapter will talk about what +they are, how they interact, and their configuration trade-offs. + + + + + + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + + + + + + Suggests how many cores each derivation should use. Similar to + make -j. + + + + +The setting determines the value of +NIX_BUILD_CORES. NIX_BUILD_CORES is equal +to , unless +equals 0, in which case NIX_BUILD_CORES +will be the total number of cores in the system. + +The total number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. + +The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Balancing 24 Build Cores
NIX_BUILD_CORESMaximum ProcessesResult
1242424 + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. +
46624 + Four derivations will be built at once, each given access to + six cores. +
126672 + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. +
241124 + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. +
24024576 + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. +
+ +It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +NIX_BUILD_CORES environment variable. + + diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 09aad2e05..4f7393e2e 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -238,8 +238,9 @@ false. linkend='opt-cores'>--cores command line switch and defaults to 1. The value 0 means that the builder should use all available CPU cores in the - system. + system. + See also . diff-hook @@ -498,7 +499,10 @@ builtins.fetchurl { regardless). It can be overridden using the () - command line switch. + command line switch. + + See also . + max-silent-time From 9031a6838ccf10b0db0d91a89e14d71a1adf6235 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Tue, 23 Jul 2019 15:21:09 +0200 Subject: [PATCH 0400/2548] Remove .github/FUNDING.yml The configuration is now done through the shared configuration repo: https://github.com/nixos/.github --- .github/FUNDING.yml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 15792e713..000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,3 +0,0 @@ -# These are supported funding model platforms - -open_collective: nixos From c82a856b36bcd008edeb9c4c981b974ae642afea Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:39:44 -0400 Subject: [PATCH 0401/2548] Add default for USER when unset uses $(id -u -n) when USER is unset, this is needed on some weird setups in Docker. Fixes #971 --- scripts/install-nix-from-closure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index fc999d336..35926f3da 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 fi -if [ -z "$USER" ]; then +if [ -z "$USER" ] && ! USER=$(id -u -n); then echo "$0: \$USER is not set" >&2 exit 1 fi From 03addc3b0a79cb7a58ae5651d1915af7383a220c Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:43:24 -0400 Subject: [PATCH 0402/2548] Use $HOME instead of $USER $USER/.nix-profile will not be a path. I think $HOME/.nix-profile was the origininal intent. /cc @Grahamc --- tests/install-darwin.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh index c99ce84ac..9933eba94 100755 --- a/tests/install-darwin.sh +++ b/tests/install-darwin.sh @@ -34,7 +34,7 @@ cleanup() { sudo rm -rf /etc/nix \ /nix \ /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ - "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels" + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" } verify() { From d171090530f4a2a79efec2c385bee1a10844c706 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 09:37:57 -0400 Subject: [PATCH 0403/2548] =?UTF-8?q?Disable=20CLONE=5FNEWUSER=20when=20it?= =?UTF-8?q?=E2=80=99s=20unavailable?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some kernels disable "unpriveleged user namespaces". This is unfortunate, but we can still use mount namespaces. Anyway, since each builder has its own nixbld user, we already have most of the benefits of user namespaces. --- src/libstore/build.cc | 14 ++++++++++++-- src/nix/run.cc | 5 ++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..c10005839 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2302,10 +2302,20 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345..90b76d666 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -199,7 +199,10 @@ void chrootHelper(int argc, char * * argv) uid_t gid = getgid(); if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot From 11d853462925d0b57fe956962e07edf5751fd4c3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 25 Jul 2019 14:29:58 -0400 Subject: [PATCH 0404/2548] Use sandbox fallback when cloning fails in builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When sandbox-fallback = true (the default), the Nix builder will fall back to disabled sandbox mode when the kernel doesn’t allow users to set it up. This prevents hard errors from occuring in tricky places, especially the initial installer. To restore the previous behavior, users can set: sandbox-fallback = false in their /etc/nix/nix.conf configuration. --- src/libstore/build.cc | 12 +++++++++++- src/libstore/globals.hh | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c10005839..dd08ce7d7 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2316,13 +2316,22 @@ void DerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(EPERM); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res == EPERM && settings.sandboxFallback) { + useChroot = false; + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2353,6 +2362,7 @@ void DerivationGoal::startBuilder() } else #endif { + fallback: options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..cc9534b27 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -209,6 +209,9 @@ public: "The paths to make available inside the build sandbox.", {"build-chroot-dirs", "build-sandbox-paths"}}; + Setting sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting extraSandboxPaths{this, {}, "extra-sandbox-paths", "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; From 13604318ad1a6ac97ef789481a550a9be65c1d48 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2019 20:06:03 +0200 Subject: [PATCH 0405/2548] epoch -> edition --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e7deb2de3..e7a85d730 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ description = "The purely functional package manager"; - epoch = 201906; + edition = 201906; inputs = [ "nixpkgs" ]; From 336afe4d5fe374569c2b13d2db90caac663573b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2019 20:09:44 +0200 Subject: [PATCH 0406/2548] nix dev-shell: Set IN_NIX_SHELL in the derivation This ensures that stdenv / setup hooks take $IN_NIX_SHELL into account. For example, stdenv only sets NIX_SSL_CERT_FILE=/no-cert-file.crt if we're not in a shell. --- src/nix/shell.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 93732f6a3..5f2724961 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -72,7 +72,7 @@ Path getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); - drv.args = {"-c", "set -e; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + drv.args = {"-c", "set -e; export IN_NIX_SHELL=impure; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); @@ -143,7 +143,6 @@ struct Common : InstallableCommand, MixProfile void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) { - out << "export IN_NIX_SHELL=1\n"; out << "nix_saved_PATH=\"$PATH\"\n"; for (auto & i : buildEnvironment.env) { From cd933b22d2041b7efc348dcc09ff255967ffc663 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Sat, 27 Jul 2019 19:40:19 -0700 Subject: [PATCH 0407/2548] Add pname and version to nix-env -q --json --- src/nix-env/nix-env.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 56ed75dae..87b2e43f0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -860,7 +860,10 @@ static void queryJSON(Globals & globals, vector & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.queryName()); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); @@ -1026,10 +1029,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.queryName(); - else if (printName) + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the From 89865144c3ba0162cd37bcbe49b3095e9bab4164 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 30 Jul 2019 11:22:55 +0200 Subject: [PATCH 0408/2548] Allow builtins.pathExists to check the existence of /nix/store paths This makes it consitent with builtins.readDir. --- src/libexpr/primops.cc | 10 ++++++++-- tests/import-derivation.nix | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 070e72f3a..350dba474 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -832,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix index 91adcd288..44fa9a45d 100644 --- a/tests/import-derivation.nix +++ b/tests/import-derivation.nix @@ -10,7 +10,10 @@ let ''; }; - value = import bar; + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; in From ee1e3132cab9d7de14837f2b04b115e544ae1622 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Tue, 30 Jul 2019 11:29:03 +0200 Subject: [PATCH 0409/2548] Disable findRuntimeRoots on darwin when running tests because lsof is slow See: https://github.com/NixOS/nix/issues/3011 --- src/libstore/gc.cc | 25 +++++++++++++++---------- tests/common.sh.in | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26e2b0dca..83cdf13a6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -444,17 +444,22 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #if !defined(__linux__) - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ } #endif diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..15d7b1ef9 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -16,6 +16,7 @@ if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home From 9a0855bbb6546e792848e551e79f8efc40782eeb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 30 Jul 2019 17:52:42 -0400 Subject: [PATCH 0410/2548] =?UTF-8?q?Don=E2=80=99t=20rely=20on=20EPERM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit startProcess does not appear to send the exit code to the helper correctly. Not sure why this is, but it is probably safe to just fallback on all sandbox errors. --- src/libstore/build.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index dd08ce7d7..0f71e7511 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2320,7 +2320,7 @@ void DerivationGoal::startBuilder() parent. This is only done when sandbox-fallback is set to true (the default). */ if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(EPERM); + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); @@ -2328,7 +2328,7 @@ void DerivationGoal::startBuilder() }, options); int res = helper.wait(); - if (res == EPERM && settings.sandboxFallback) { + if (res != 0 && settings.sandboxFallback) { useChroot = false; goto fallback; } else if (res != 0) From 320126aeebc89f8c34ee1668ebacf1fe4b512ed8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 14:04:09 +0200 Subject: [PATCH 0411/2548] Tweak min-free/max-free descriptions --- doc/manual/command-ref/conf-file.xml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 4f7393e2e..0a97719ef 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -483,8 +483,10 @@ builtins.fetchurl { max-free - This option defines after how many free bytes to stop collecting - garbage once the min-free condition gets triggered. + When a garbage collection is triggered by the + min-free option, it stops as soon as + max-free bytes are available. The default is + infinity (i.e. delete all garbage). @@ -528,9 +530,11 @@ builtins.fetchurl { min-free - When the disk reaches min-free bytes of free disk space during a build, nix - will start to garbage-collection until max-free bytes are available on the disk. - A value of 0 (the default) means that this feature is disabled. + When free disk space in /nix/store + drops below min-free during a build, Nix + performs a garbage-collection until max-free + bytes are available or there is no more garbage. A value of + 0 (the default) disables this feature. From 7c5596734f65b30b250ea73a423b40a4ce513fdf Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 11 Jul 2019 20:23:03 +0200 Subject: [PATCH 0412/2548] Add a post-build-hook Passing `--post-build-hook /foo/bar` to a nix-* command will cause `/foo/bar` to be executed after each build with the following environment variables set: DRV_PATH=/nix/store/drv-that-has-been-built.drv OUT_PATHS=/nix/store/...build /nix/store/...build-bin /nix/store/...build-dev This can be useful in particular to upload all the builded artifacts to the cache (including the ones that don't appear in the runtime closure of the final derivation or are built because of IFD). This new feature prints the stderr/stdout output to the `nix-build` and `nix build` client, and the output is printed in a Nix 2 compatible format: [nix]$ ./inst/bin/nix-build ./test.nix these derivations will be built: /nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv building '/nix/store/ishzj9ni17xq4hgrjvlyjkfvm00b0ch9-my-example-derivation.drv'... hello! bye! running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... post-build-hook: + sleep 1 post-build-hook: + echo 'Signing paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Signing paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + echo 'Uploading paths' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: Uploading paths /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation post-build-hook: + sleep 1 post-build-hook: + printf 'very important stuff' /nix/store/qr213vjmibrqwnyp5fw678y7whbkqyny-my-example-derivation [nix-shell:~/projects/github.com/NixOS/nix]$ ./inst/bin/nix build -L -f ./test.nix my-example-derivation> hello! my-example-derivation> bye! my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Signing paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Signing paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + echo 'Uploading paths' /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> Uploading paths /nix/store/c263gzj2kb2609mz8wrbmh53l14wzmfs-my-example-derivation my-example-derivation (post)> + sleep 1 my-example-derivation (post)> + printf 'very important stuff' [1 built, 0.0 MiB DL] Co-authored-by: Graham Christensen Co-authored-by: Eelco Dolstra --- .../advanced-topics/advanced-topics.xml | 1 + .../advanced-topics/post-build-hook.xml | 157 ++++++++++++++++++ doc/manual/command-ref/conf-file.xml | 55 ++++++ src/libstore/build.cc | 55 ++++++ src/libstore/globals.hh | 3 + src/libutil/logging.hh | 2 + src/libutil/util.cc | 23 ++- src/libutil/util.hh | 2 + src/nix/progress-bar.cc | 16 +- tests/dependencies.nix | 1 + tests/local.mk | 3 +- tests/post-hook.sh | 15 ++ tests/push_to_store.sh | 4 + 13 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 doc/manual/advanced-topics/post-build-hook.xml create mode 100644 tests/post-hook.sh create mode 100755 tests/push_to_store.sh diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..1b8841ad2 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -8,5 +8,6 @@ + diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..4335b308b --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,157 @@ + + +Using the <xref linkend="conf-post-build-hook" /> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +we will restart the Nix daemon a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will only contain characters which are safe for word + splitting, and free of any globs. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 09aad2e05..d2c9c7502 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -656,6 +656,61 @@ password my-password + + post-build-hook + + Optional. The path to a program to execute after each build. + + This option is only settable in the global + nix.conf, or on the command line by trusted + users. + + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. + + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + + + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space ( ) character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + repeat How many times to repeat builds to check whether diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..7494eec41 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..7dea68921 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -315,6 +315,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f2219445..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -26,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -36,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5c..44fa72482 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -84,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8d..b538a0b41 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -270,12 +270,14 @@ struct RunOptions std::optional uid; std::optional gid; std::optional chdir; + std::optional> environment; Path program; bool searchPath = true; Strings args; std::optional input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index b1c1d87de..c0bcfb0c9 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -170,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -228,14 +236,18 @@ public: update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add..eca4b2964 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..8daaa859f 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -26,7 +26,8 @@ nix_tests = \ check.sh \ plugins.sh \ search.sh \ - nix-copy-ssh.sh + nix-copy-ssh.sh \ + post-hook.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 000000000..3f82d0c5e --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push_to_store.sh b/tests/push_to_store.sh new file mode 100755 index 000000000..d97eb095d --- /dev/null +++ b/tests/push_to_store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs From ec415d7166d607c92cf8f1af688f86e4b4731dff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 17:07:33 +0200 Subject: [PATCH 0413/2548] Add a test for auto-GC This currently fails because we're using POSIX file locks. So when the garbage collector opens and closes its own temproots file, it causes the lock to be released and then deleted by another GC instance. --- src/libstore/gc.cc | 9 +++++-- src/libstore/globals.hh | 3 +++ tests/gc-auto.sh | 59 +++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 4 ++- 4 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 tests/gc-auto.sh diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 83cdf13a6..eeb237839 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -871,7 +871,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -892,7 +897,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..0c0a9ec54 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -342,6 +342,9 @@ public: Setting maxFree{this, std::numeric_limits::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 000000000..1e91282d0 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,59 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) +garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) +garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +expr2=$(cat < \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..261e4db0d 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,9 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - gc.sh gc-concurrent.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ From e349f2c0a370e4dfd09ae51c2cae4db08a834ad5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Oct 2018 15:03:03 +0200 Subject: [PATCH 0414/2548] Use BSD instead of POSIX file locks POSIX file locks are essentially incompatible with multithreading. BSD locks have much saner semantics. We need this now that there can be multiple concurrent LocalStore::buildPaths() invocations. --- src/libstore/build.cc | 17 ----- src/libstore/gc.cc | 4 +- src/libstore/local-store.hh | 2 +- src/libstore/pathlocks.cc | 125 ++++++++++++------------------------ src/libstore/pathlocks.hh | 4 -- 5 files changed, 45 insertions(+), 107 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..a28619ab9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3984,17 +3984,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); worker.updateProgress(); @@ -4034,12 +4023,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index eeb237839..2c791efbe 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6b655647b..af8b84bf5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -263,7 +263,7 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..b6d8547c7 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -7,7 +7,7 @@ #include #include -#include +#include namespace nix { @@ -40,17 +40,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +55,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +67,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +80,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +88,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } + AutoCloseFD fd; - try { + while (1) { - AutoCloseFD fd; + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - while (1) { - - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); - - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); + debug(format("lock acquired on '%1%'") % lockPath); - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +157,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +174,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } From a2597d5f27bc6cfa26343be2f481c8a9d1e22753 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Aug 2019 18:37:55 +0200 Subject: [PATCH 0415/2548] Simplify With BSD locks we don't have to guard against reading our own temproots. --- src/libstore/gc.cc | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 2c791efbe..366dbfb0a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); From 399b6f3c46077e10a7047e8216fc1a67425a768a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Oct 2018 14:46:30 +0200 Subject: [PATCH 0416/2548] nix-store --verify: Don't repair while holding the GC lock --- src/libstore/local-store.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..63b11467e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1210,7 +1210,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1221,13 +1222,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); From 363a2f68261af73aefe4edead9c0526030751a27 Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 6 Aug 2019 14:26:43 -0400 Subject: [PATCH 0417/2548] post-build-hook: docs fixup --- doc/manual/advanced-topics/post-build-hook.xml | 7 +++++-- doc/manual/command-ref/conf-file.xml | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 4335b308b..3dc43ee79 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -74,6 +74,8 @@ trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDS #!/bin/sh set -eu +set -f # disable globbing +export IFS=' ' echo "Signing paths" $OUT_PATHS nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS @@ -88,8 +90,9 @@ exec nix copy --to 's3://example-nix-cache' $OUT_PATHS list of Nix store paths. In this case, we expect and want the shell to perform word splitting to make each output path its own argument to nix sign-paths. Nix guarantees - the paths will only contain characters which are safe for word - splitting, and free of any globs. + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index d2c9c7502..e818a74cd 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -674,6 +674,7 @@ password my-password The hook does not execute on substituted paths. The hook's output always goes to the user's terminal. If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. The program executes with no arguments. The program's environment @@ -693,7 +694,7 @@ password my-password OUT_PATHS - Output paths of the built derivation, separated by a space ( ) character. + Output paths of the built derivation, separated by a space character. Example: /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc From c3fefd1a6efec457395a187d15f435447dee6f3b Mon Sep 17 00:00:00 2001 From: Will Dietz Date: Wed, 7 Aug 2019 07:34:11 -0500 Subject: [PATCH 0418/2548] pathlocks: add include to fcntl.h for O_CLOEXEC --- src/libstore/pathlocks.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index b6d8547c7..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include #include From 1eeaf99cf89630f84b4a1b6882a0f98019725d9e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 7 Aug 2019 14:45:07 -0400 Subject: [PATCH 0419/2548] fixup: docs for post-build-hook --- doc/manual/command-ref/conf-file.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index f3c721c78..c7b540892 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -682,7 +682,7 @@ password my-password The hook does not execute on substituted paths. The hook's output always goes to the user's terminal. If the hook fails, the build succeeds but no further builds execute. - The hook executes synchronously, and blocks other builds from progressing while it runs. + The hook executes synchronously, and blocks other builds from progressing while it runs. The program executes with no arguments. The program's environment From 2053ac774797f237a5335c4232bc1ed8902686cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Aug 2019 12:18:46 +0200 Subject: [PATCH 0420/2548] Rename file for consistency --- tests/post-hook.sh | 2 +- tests/{push_to_store.sh => push-to-store.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{push_to_store.sh => push-to-store.sh} (100%) diff --git a/tests/post-hook.sh b/tests/post-hook.sh index 3f82d0c5e..54d605d63 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -5,7 +5,7 @@ clearStore export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store -nix-build dependencies.nix --post-build-hook $PWD/push_to_store.sh +nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh clearStore diff --git a/tests/push_to_store.sh b/tests/push-to-store.sh similarity index 100% rename from tests/push_to_store.sh rename to tests/push-to-store.sh From 05a10dd835923092b54fcad2276d54bf164c1b7f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Aug 2019 14:39:05 +0200 Subject: [PATCH 0421/2548] tests/post-hook.sh: Don't put result link in cwd --- tests/post-hook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/post-hook.sh b/tests/post-hook.sh index 54d605d63..a02657215 100644 --- a/tests/post-hook.sh +++ b/tests/post-hook.sh @@ -5,7 +5,7 @@ clearStore export REMOTE_STORE=$TEST_ROOT/remote_store # Build the dependencies and push them to the remote store -nix-build dependencies.nix --post-build-hook $PWD/push-to-store.sh +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh clearStore From ab16b3d076e9cd3ecfdcde128f43dd486b072557 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 8 Aug 2019 14:32:02 +0200 Subject: [PATCH 0422/2548] Fix gc-auto test --- tests/gc-auto.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 1e91282d0..e770e665c 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -29,7 +29,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-A -L "($expr)" \ +nix build --impure -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & pid=$! @@ -50,7 +50,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-B -L "($expr2)" \ +nix build --impure -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 wait "$pid" From a02457db715a80de7630d823fd8d7230a4bd451f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Wed, 7 Aug 2019 15:14:40 -0400 Subject: [PATCH 0423/2548] conf: stalled-download-timeout: make tunable Make curl's low speed limit configurable via stalled-download-timeout. Before, this limit was five minutes without receiving a single byte. This is much too long as if the remote end may not have even acknowledged the HTTP request. --- doc/manual/command-ref/conf-file.xml | 8 ++++++++ src/libstore/download.cc | 4 +--- src/libstore/download.hh | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c7b540892..1325a3f22 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -873,6 +873,14 @@ password my-password + stalled-download-timeout + + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration. + + + substituters A list of URLs of substituters, separated by diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 91087eebc..c322d267d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 3b7fff3ba..c68381846 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -24,6 +24,9 @@ struct DownloadSettings : Config Setting connectTimeout{this, 0, "connect-timeout", "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + Setting stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; }; From 662db921e2f5bebeb0fd455aa744708468df8bfa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Aug 2019 18:51:52 +0200 Subject: [PATCH 0424/2548] nix dev-shell: Set dontAddDisableDepTrack --- src/nix/shell.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 5f2724961..a3827c297 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -72,7 +72,7 @@ Path getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); - drv.args = {"-c", "set -e; export IN_NIX_SHELL=impure; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + drv.args = {"-c", "set -e; export IN_NIX_SHELL=impure; export dontAddDisableDepTrack=1; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); From ee9c988a1b2e3c511b8613e698a0f9632ab1538f Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Fri, 12 Apr 2019 12:31:33 -0400 Subject: [PATCH 0425/2548] Track function start and ends for flame graphs With this patch, and this file I called `log.py`: #!/usr/bin/env nix-shell #!nix-shell -i python3 -p python3 --pure import sys from pprint import pprint stack = [] timestack = [] for line in open(sys.argv[1]): components = line.strip().split(" ", 2) if components[0] != "function-trace": continue direction = components[1] components = components[2].rsplit(" ", 2) loc = components[0] _at = components[1] time = int(components[2]) if direction == "entered": stack.append(loc) timestack.append(time) elif direction == "exited": dur = time - timestack.pop() vst = ";".join(stack) print(f"{vst} {dur}") stack.pop() and: nix-instantiate --trace-function-calls -vvvv ../nixpkgs/pkgs/top-level/release.nix -A unstable > log.matthewbauer 2>&1 ./log.py ./log.matthewbauer > log.matthewbauer.folded flamegraph.pl --title matthewbauer-post-pr log.matthewbauer.folded > log.matthewbauer.folded.svg I can make flame graphs like: http://gsc.io/log.matthewbauer.folded.svg --- Includes test cases around function call failures and tryEval. Uses RAII so the finish is always called at the end of the function. --- contrib/stack-collapse.py | 39 +++++++++++++ doc/manual/command-ref/conf-file.xml | 28 +++++++++ src/libexpr/eval.cc | 8 ++- src/libexpr/eval.hh | 4 ++ src/libexpr/function-trace.hh | 24 ++++++++ tests/function-trace.sh | 86 ++++++++++++++++++++++++++++ tests/local.mk | 3 +- 7 files changed, 189 insertions(+), 3 deletions(-) create mode 100755 contrib/stack-collapse.py create mode 100644 src/libexpr/function-trace.hh create mode 100755 tests/function-trace.sh diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 000000000..ee77e1589 --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `-vvvv` and +# `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls -vvvv '' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c7b540892..407e5d10b 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -973,6 +973,34 @@ requiredSystemFeatures = [ "kvm" ]; + trace-function-calls + + + + Default: false. + + If set to true, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit. + + +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + + The undefined position means the function + call is a builtin. + + Use the contrib/stack-collapse.py script + distributed with the Nix source code to convert the trace logs + in to a format suitable for flamegraph.pl. + + + + + trusted-public-keys A whitespace-separated list of public keys. When diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..9f4b6b411 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "json.hh" #include +#include #include #include #include @@ -16,7 +17,6 @@ #include #include -#include #include #if HAVE_BOEHMGC @@ -1094,9 +1094,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..22472fd72 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include #include @@ -349,6 +350,9 @@ struct EvalSettings : Config Setting allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8234b7603 --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + vomit("function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + vomit("function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 000000000..f7d93b435 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,86 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + -vvvv \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/local.mk b/tests/local.mk index ef359c631..187f96ea2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -29,7 +29,8 @@ nix_tests = \ plugins.sh \ search.sh \ nix-copy-ssh.sh \ - post-hook.sh + post-hook.sh \ + function-trace.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 653c40778462b5921fbbe2abffd1a2822f08ec57 Mon Sep 17 00:00:00 2001 From: Daniel Diaz Date: Fri, 14 Jun 2019 00:05:31 +0200 Subject: [PATCH 0426/2548] Expanded documentation for .nix-defexpr --- doc/manual/command-ref/nix-env.xml | 40 +++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 56c466268..693f23f7f 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -221,31 +221,53 @@ also . ~/.nix-defexpr - A directory that contains the default Nix + The source for the default Nix expressions used by the , , and operations to obtain derivations. The + --available operations to obtain derivations. The option may be used to override this default. - The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if ~/.nix-defexpr contains - two files, foo and bar, + If ~/.nix-defexpr is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression. + + If ~/.nix-defexpr is a directory + containing a default.nix file, that file + is loaded as in the above paragraph. + + If ~/.nix-defexpr is a directory without + a default.nix file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + + + For example, if ~/.nix-defexpr contains + two files, foo.nix and bar.nix, then the default Nix expression will essentially be { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; } + The file manifest.nix is always ignored. + Subdirectories without a default.nix file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression. + The command nix-channel places symlinks to the downloaded Nix expressions from each subscribed channel in this directory. - From 91b00b145f0c50e346d0250168cbcbcba7aa3b40 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 12 Aug 2019 21:03:48 +0200 Subject: [PATCH 0427/2548] libutil: add SizedSource Introduce the SizeSource which allows to bound how much data is being read from a source. It also contains a drainAll() function to discard the rest of the source, useful to keep the nix protocol in sync. --- src/libutil/serialise.hh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 969e4dff3..a344a5ac7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink From b226b5cd976ca71abb3c0861b56d5e5940430924 Mon Sep 17 00:00:00 2001 From: zimbatm Date: Mon, 12 Aug 2019 17:19:43 +0200 Subject: [PATCH 0428/2548] nix-store: fix out of sync protocol If a NAR is already in the store, addToStore doesn't read the source which makes the protocol go out of sync. This happens for example when two client try to nix-copy-closure the same derivation at the same time. --- src/nix-store/nix-store.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f324056bb..0cbceb02f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success From 1dbaf119486fecbdf57ecc2f877307030b665f69 Mon Sep 17 00:00:00 2001 From: Toon Nolten Date: Sat, 17 Aug 2019 16:33:35 +0200 Subject: [PATCH 0429/2548] Fix nix-env documentation for --delete-generations The documentation for `--delete-generations` had an erroneous fullstop and as it turns out inaccurate information on the `+No.` syntax. --- doc/manual/command-ref/nix-env.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 693f23f7f..9d13d3014 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1370,10 +1370,10 @@ profile. The generations can be a list of generation numbers, the special value old to delete all non-current generations, a value such as 30d to delete all generations older than the specified number of days (except for the -generation that was active at that point in time), or a value such as. -+5 to only keep the specified items older than the -current generation. Periodically deleting old generations is important -to make garbage collection effective. +generation that was active at that point in time), or a value such as ++5 to only keep the specified number of generations. +Periodically deleting old generations is important to make garbage collection +effective. From 0463d5e36ffdbb60f851bd3b34756eff84151b28 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 22 Aug 2019 23:38:52 -0400 Subject: [PATCH 0430/2548] Allow empty /nix directory in multi-user installer With macOS catalina, we can no longer modify the root system volume (#2925). macOS provides a system configuration file in synthetic.conf(5) to create empty root directories. This can be used to mount /nix to a separate volume. As a result, this directory will need to already exist prior to installation. Instead, check for /nix/store and /nix/var for a live Nix installation. --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9b757e7da..3d8b035eb 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -330,7 +330,7 @@ EOF fi done - if [ -d /nix ]; then + if [ -d /nix/store ] || [ -d /nix/var ]; then failure < Date: Fri, 23 Aug 2019 15:50:54 -0400 Subject: [PATCH 0431/2548] operators: document exact precedenc, split up similar operators --- doc/manual/expressions/language-operators.xml | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index f1f750934..1b14bc658 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -15,13 +15,16 @@ weakest binding). + Name Syntax Associativity Description + Precedence + Select e . attrpath [ or def ] @@ -33,19 +36,25 @@ weakest binding). dot-separated list of attribute names.) If the attribute doesn’t exist, return def if provided, otherwise abort evaluation. + 1 + Application e1 e2 left Call function e1 with argument e2. + 2 + Arithmetic Negation - e none Arithmetic negation. + 3 + Has Attribute e ? attrpath none @@ -53,34 +62,69 @@ weakest binding). the attribute denoted by attrpath; return true or false. + 4 + List Concatenation e1 ++ e2 right List concatenation. + 5 + Multiplication e1 * e2, + + left + Arithmetic multiplication. + 6 + + + Division + e1 / e2 left - Arithmetic multiplication and division. + Arithmetic division. + 6 + Addition + + e1 + e2 + + left + Arithmetic addition. + 7 + + + Subtraction - e1 + e2, e1 - e2 left - Arithmetic addition and subtraction. String or path concatenation (only by +). + Arithmetic subtraction. + 7 + String Concatenation + + string1 + string2 + + left + String concatenation. + 7 + + + Not ! e none Boolean negation. + 8 + Update e1 // e2 right @@ -89,47 +133,90 @@ weakest binding). e2 (with the latter taking precedence over the former in case of equally named attributes). + 9 + Less Than e1 < e2, - e1 > e2, - e1 <= e2, + + none + Arithmetic comparison. + 10 + + + Less Than or Equal To + + e1 <= e2 + + none + Arithmetic comparison. + 10 + + + Greater Than + + e1 > e2 + + none + Arithmetic comparison. + 10 + + + Greater Than or Equal To + e1 >= e2 none Arithmetic comparison. + 10 + Equality + + e1 == e2 + + none + Equality. + 11 + + + Inequality - e1 == e2, e1 != e2 none - Equality and inequality. + Inequality. + 11 + Logical And e1 && e2 left Logical AND. + 12 + Logical Or e1 || e2 left Logical OR. + 13 + Logical Implication e1 -> e2 none Logical implication (equivalent to !e1 || e2). + 14 -
\ No newline at end of file + From 5c06a8d3283139140e765b5f10ad7102a6a3e964 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 23 Aug 2019 20:24:39 -0400 Subject: [PATCH 0432/2548] Reset tmpDirInSandbox for unsandboxed --- src/libstore/build.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0f71e7511..96e9b8edd 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2330,6 +2330,7 @@ void DerivationGoal::startBuilder() int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { useChroot = false; + tmpDirInSandbox = tmpDir; goto fallback; } else if (res != 0) throw Error("unable to start build process"); From 6dab42a551a769f304c2a0e66e8771d7289502f2 Mon Sep 17 00:00:00 2001 From: Venkateswara Rao Mandela Date: Sat, 15 Jun 2019 00:27:26 +0530 Subject: [PATCH 0433/2548] installer: handle network proxy in systemd install If a network proxy configuration is detected, setup an override systemd unit file for nix-daemon service with the non-empty proxy variables. Proxy detection is performed by looking for http/https/ftp proxy and no proxy variables in user environment --- scripts/install-systemd-multi-user.sh | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) mode change 100644 => 100755 scripts/install-systemd-multi-user.sh diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh old mode 100644 new mode 100755 index 04bc539a1..bef3ac4f9 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat < Date: Fri, 23 Aug 2019 16:02:49 -0400 Subject: [PATCH 0434/2548] docs: document the installer's use of proxy env vars --- doc/manual/installation/env-variables.xml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index d1ee0bb2e..e2b8fc867 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon +
+ +Proxy Environment Variables + +The Nix installer has special handling for these proxy-related +environment variables: +http_proxy, https_proxy, +ftp_proxy, no_proxy, +HTTP_PROXY, HTTPS_PROXY, +FTP_PROXY, NO_PROXY. + +If any of these variables are set when running the Nix installer, +then the installer will create an override file at +/etc/systemd/system/nix-daemon.service.d/override.conf +so nix-daemon will use them. + +
+ From d45922472434783bdc29610479a735dd236229a3 Mon Sep 17 00:00:00 2001 From: Piotr Szubiakowski Date: Tue, 27 Aug 2019 10:35:35 +0200 Subject: [PATCH 0435/2548] nix-daemon.service: add install section. Signed-off-by: Piotr Szubiakowski --- misc/systemd/nix-daemon.service.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 5fc04a3f5..25655204d 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process + +[Install] +WantedBy=multi-user.target From 171d784404fefa7c91e2082f92ec92c39cc1aa8e Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 27 Aug 2019 06:55:22 -0400 Subject: [PATCH 0436/2548] docs: operators: Make OR and AND capitalized --- doc/manual/expressions/language-operators.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index 1b14bc658..4f11bf529 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -190,7 +190,7 @@ weakest binding). 11 - Logical And + Logical AND e1 && e2 left @@ -198,7 +198,7 @@ weakest binding). 12 - Logical Or + Logical OR e1 || e2 left From 800fba1037ad97ff5e6d9acf9e820a30f0922f59 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 27 Aug 2019 10:57:09 -0400 Subject: [PATCH 0437/2548] Use wait4path on org.nixos.nix-daemon.plist When using a volume, the nix-daemon path may not exist. To avoid this issue, we must use the wait4path tool. This should solve one of the issues in multi-user on macOS Catalina. --- misc/launchd/org.nixos.nix-daemon.plist.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 92ed12fa5..6bc448d28 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -14,7 +14,12 @@ RunAtLoad Program - @bindir@/nix-daemon + /bin/sh + ProgramArguments + + -c + /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon + StandardErrorPath /var/log/nix-daemon.log StandardOutPath From 45b3dc325a1d49f86182195cf22fc96ddc05c644 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 17:00:04 +0200 Subject: [PATCH 0438/2548] Add 2.3 release notes --- doc/manual/release-notes/rl-2.3.xml | 70 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 428213b36..b0f7c8ec5 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,9 +4,23 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -Release 2.3 (????-??-??) +Release 2.3 (2019-08-??) -This release contains the following changes: +This is primarily a bug fix release. However, it makes some +incompatible changes: + + + + + Nix now uses BSD file locks instead of POSIX file + locks. Since previous releases used POSIX file locks, you should + not use Nix 2.2 and previous releases at the same time on a Nix + store. + + + + +It also has the following changes: @@ -18,5 +32,57 @@ already begin with refs/. + + + The installer now enables sandboxing by default on + Linux. The max-jobs setting now defaults to + 1. + + + + New builtin functions: + builtins.isPath, + builtins.hashFile. + + + + + nix: Add + () flag to + print build log output to stderr rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + + + + + Builds are now executed in a pseudo-terminal, and the + TERM evnironment variable is set to + xterm-256color. This allows many programs + (e.g. gcc, clang, + cmake) to print colorized log output. + + + + Add convenience flag. This flag + disables substituters; sets the tarball-ttl + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces. + + + + Add a post-build-hook setting to run a + program after a build has succeeded. + + + + Add a trace-function-calls setting to log + the duration of Nix function calls to stderr. + + + From 73728874ab89d77bce3b103acb239d4d5eec14b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 17:01:54 +0200 Subject: [PATCH 0439/2548] Hopefully fix post-hook test on macOS https://hydra.nixos.org/build/99262744 --- tests/push-to-store.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh index d97eb095d..6aadb916b 100755 --- a/tests/push-to-store.sh +++ b/tests/push-to-store.sh @@ -1,4 +1,4 @@ #!/bin/sh echo Pushing "$@" to "$REMOTE_STORE" -echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs From fdff96501f7f1762446a930e3a27b56a4fc912c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 21:18:00 +0200 Subject: [PATCH 0440/2548] Update release notes --- doc/manual/release-notes/rl-2.3.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index b0f7c8ec5..a1f4a4b75 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -83,6 +83,11 @@ incompatible changes: the duration of Nix function calls to stderr. + + On Linux, sandboxing is now disabled by default on systems + that don’t have the necessary kernel support. + + From ceefddafe839a0c9ac553367e16f965dda7fb086 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 22:18:34 +0200 Subject: [PATCH 0441/2548] Compress binary tarballs using xz Fixes https://github.com/NixOS/nix/issues/240. Apparently 'tar -xf' can decompress xz files on macOS nowadays. --- release.nix | 6 +++--- scripts/install.in | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/release.nix b/release.nix index 78b39108f..bec955730 100644 --- a/release.nix +++ b/release.nix @@ -165,10 +165,10 @@ let chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} - fn=$out/$dir.tar.bz2 + fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfj $fn \ + tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --absolute-names \ --hard-dereference \ @@ -295,7 +295,7 @@ let substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ") + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${build.x86_64-linux.src.version} diff --git a/scripts/install.in b/scripts/install.in index 4857638c0..902758b13 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -30,12 +30,11 @@ case "$(uname -s).$(uname -m)" in *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2" +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")" +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" -require_util bzcat "decompress the binary tarball" require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." @@ -57,7 +56,7 @@ fi unpack=$tmpDir/unpack mkdir -p "$unpack" -< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) From 5fa8b3f96577db65e080aaef65ad1cae462356cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 27 Aug 2019 22:38:48 +0200 Subject: [PATCH 0442/2548] Update the libboost hack This cuts about 46 MiB from the closure. --- release.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release.nix b/release.nix index bec955730..ab677f3bd 100644 --- a/release.nix +++ b/release.nix @@ -72,7 +72,12 @@ let # https://github.com/NixOS/nixpkgs/issues/45462 '' mkdir -p $out/lib - cp ${boost}/lib/libboost_context* $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + chmod u+w $out/lib/*.so.* + ${lib.optionalString stdenv.isLinux '' + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} ''; configureFlags = configureFlags ++ From 7298a38a07d62d2077ca30a3ff619e32edb4eb1e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 16:29:44 +0200 Subject: [PATCH 0443/2548] Don't send certain setting overrides to the daemon These are already handled separately. This fixes warnings like warning: ignoring the user-specified setting 'max-jobs', because it is a restricted setting and you are not a trusted user when using the -j flag. --- src/libstore/remote-store.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 15faf78a5..1c2e23f9c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -191,6 +191,13 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map overrides; globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 693e68e09c9a17ca72b074bd2e575bf435647b45 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 28 Aug 2019 12:26:08 -0400 Subject: [PATCH 0444/2548] Set maximum name length in Nix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously we allowed any length of name for Nix derivations. This is bad because different file systems have different max lengths. To make things predictable, I have picked a max. This was done by trying to build this derivation: derivation { name = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; builder = "/no-such-path"; system = "x86_64-linux"; } Take off one a and it will not lead to file name too long. That ends up being 212 a’s. An even smaller max could be picked if we want to support more file systems. Working backwards, this is why: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-${name}.drv.chroot > 255 - 32 - 1 - 4 - 7 = 211 --- src/libstore/store-api.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f5608d384..3bb9db0b7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -97,6 +97,10 @@ void checkStoreName(const string & name) reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || From c128031492b7134f67065b66a89f9183e79e99f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 22:04:16 +0200 Subject: [PATCH 0445/2548] Fix macOS build https://hydra.nixos.org/build/99500938 --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index ab677f3bd..0fe683518 100644 --- a/release.nix +++ b/release.nix @@ -74,8 +74,8 @@ let mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - chmod u+w $out/lib/*.so.* ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} ''; From b6120d26a84a69755c049ca5115b4388239d4540 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Aug 2019 22:19:31 +0200 Subject: [PATCH 0446/2548] gc-auto.sh: Increase verbosity --- tests/gc-auto.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 1e91282d0..4b1f6374a 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -29,7 +29,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-A -L "($expr)" \ +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & pid=$! @@ -50,7 +50,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -o $TEST_ROOT/result-B -L "($expr2)" \ +nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 wait "$pid" From f27e53f77e39d1247c47b48acb63af3d24ca74bf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:09:58 +0200 Subject: [PATCH 0447/2548] Cleanup --- src/libstore/gc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 366dbfb0a..f73601679 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -920,11 +920,11 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); - GCOptions options; options.maxFreed = settings.maxFree - avail; + printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCResults results; collectGarbage(options, results); From ecb0a23d51e46e2e0180850bdc8ad6cb7cc3d13f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:10:01 +0200 Subject: [PATCH 0448/2548] Add some more instrumentation --- tests/gc-auto.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 4b1f6374a..49894bc68 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -6,6 +6,9 @@ garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) +ls -l $garbage3 +du $garbage3 + fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free echo 1100 > $fake_free From 31f5ecfaa5bce2282f50daed1e4f477c64705fb3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 12:35:15 +0200 Subject: [PATCH 0449/2548] Maybe fix #3058 --- tests/gc-auto.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 49894bc68..68b7219f2 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -2,12 +2,12 @@ source common.sh clearStore -garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) -garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) -garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) ls -l $garbage3 -du $garbage3 +POSIXLY_CORRECT=1 du $garbage3 fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free From 5bdac86be2eb6d541784c4f149bfa06b59ef63a2 Mon Sep 17 00:00:00 2001 From: toonn Date: Thu, 29 Aug 2019 13:46:38 +0200 Subject: [PATCH 0450/2548] Reword to clarify newer generations are left alone My attempt at clarifying the docs resulted in a false explanation. This is now fixed and I added an example to eliminate all possible confusion. --- doc/manual/command-ref/nix-env.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 9d13d3014..c8c01f9a4 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -1371,7 +1371,10 @@ special value old to delete all non-current generations, a value such as 30d to delete all generations older than the specified number of days (except for the generation that was active at that point in time), or a value such as -+5 to only keep the specified number of generations. ++5 to keep the last 5 generations +ignoring any newer than current, e.g., if 30 is the current +generation +5 will delete generation 25 +and all older generations. Periodically deleting old generations is important to make garbage collection effective. From a2c4fcd5e9782dc8d2998773380c7171ee53b813 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Aug 2019 14:49:58 +0200 Subject: [PATCH 0451/2548] Don't rely on st_blocks It doesn't seem very reliable on ZFS. --- src/libstore/gc.cc | 7 +++---- src/libutil/util.cc | 2 +- tests/gc-auto.sh | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f73601679..a166f4ee2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -690,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting '%1%'") % path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } @@ -701,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_blocks * 512ULL; + state.results.bytesFreed += st.st_size; } struct stat st; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 44fa72482..1b7449991 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -397,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 68b7219f2..54ead227e 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -22,7 +22,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... @@ -44,7 +44,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... From 89468410d56d2b7b86ac9d60d70757f4d8f33718 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 11:22:34 +0200 Subject: [PATCH 0452/2548] Extract flake dependencies from the 'outputs' arguments That is, instead of inputs = [ "nixpkgs" ]; outputs = inputs: ... inputs.nixpkgs ...; you can write outputs = { nixpkgs }: ... inputs.nixpkgs ...; --- src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 2 +- src/libexpr/flake/flake.cc | 17 +++++++++++++---- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index faa76f1f7..fa79b0d5e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -304,6 +304,7 @@ EvalState::EvalState(const Strings & _searchPath, ref store) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) , sDescription(symbols.create("description")) + , sSelf(symbols.create("self")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 75e91e6b1..5e976f196 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -80,7 +80,7 @@ public: sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sDescription; + sDescription, sSelf; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index e6cef502c..eb40f3b2a 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -232,10 +232,10 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (edition) { flake.edition = state.forceInt(*(**edition).value, *(**edition).pos); - if (flake.edition < 201906) - throw Error("flake '%s' has illegal edition %d", flakeRef, flake.edition); - if (flake.edition > 201906) + if (flake.edition > 201909) throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", flakeRef, flake.edition); + if (flake.edition < 201909) + throw Error("flake '%s' has illegal edition %d", flakeRef, flake.edition); } else throw Error("flake '%s' lacks attribute 'edition'", flakeRef); @@ -272,6 +272,15 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (auto outputs = vInfo.attrs->get(sOutputs)) { state.forceFunction(*(**outputs).value, *(**outputs).pos); flake.vOutputs = (**outputs).value; + + if (flake.vOutputs->lambda.fun->matchAttrs) { + for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) { + flake.inputs.push_back(FlakeRef(formal.name)); + } + } + } + } else throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); @@ -538,7 +547,7 @@ void callFlake(EvalState & state, auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); mkApp(*vOutputs, *flake.vOutputs, v); - v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + v.attrs->push_back(Attr(state.sSelf, &v)); v.attrs->sort(); From 2341f30ec66ca6b6e795eb9764ec7a0496c1c4f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 13:06:23 +0200 Subject: [PATCH 0453/2548] Clean up the 'outputs' interface --- src/libexpr/flake/flake.cc | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index eb40f3b2a..2a6792af9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -505,17 +505,14 @@ void callFlake(EvalState & state, const FlakeInputs & inputs, Value & vRes) { - // Construct the resulting attrset '{outputs, ...}'. This attrset - // is passed lazily as an argument to the 'outputs' function. + auto & vInputs = *state.allocValue(); - auto & v = *state.allocValue(); - - state.mkAttrs(v, + state.mkAttrs(vInputs, inputs.flakeInputs.size() + - inputs.nonFlakeInputs.size() + 8); + inputs.nonFlakeInputs.size() + 1); for (auto & dep : inputs.flakeInputs) { - auto vFlake = state.allocAttr(v, dep.second.id); + auto vFlake = state.allocAttr(vInputs, dep.second.id); auto vPrimOp = state.allocValue(); static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); vPrimOp->type = tPrimOp; @@ -528,7 +525,7 @@ void callFlake(EvalState & state, } for (auto & dep : inputs.nonFlakeInputs) { - auto vNonFlake = state.allocAttr(v, dep.first); + auto vNonFlake = state.allocAttr(vInputs, dep.first); auto vPrimOp = state.allocValue(); static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); vPrimOp->type = tPrimOp; @@ -540,24 +537,27 @@ void callFlake(EvalState & state, mkApp(*vNonFlake, *vPrimOp, *vArg); } - mkString(*state.allocAttr(v, state.sDescription), flake.description); + auto & vSourceInfo = *state.allocValue(); + state.mkAttrs(vSourceInfo, 8); + emitSourceInfoAttrs(state, flake.sourceInfo, vSourceInfo); - emitSourceInfoAttrs(state, flake.sourceInfo, v); + vInputs.attrs->push_back(Attr(state.sSelf, &vRes)); - auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); - mkApp(*vOutputs, *flake.vOutputs, v); - - v.attrs->push_back(Attr(state.sSelf, &v)); - - v.attrs->sort(); + vInputs.attrs->sort(); /* For convenience, put the outputs directly in the result, so you can refer to an output of an input as 'inputs.foo.bar' rather than 'inputs.foo.outputs.bar'. */ - auto v2 = *state.allocValue(); - state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + auto vCall = *state.allocValue(); + state.eval(state.parseExprFromString( + "outputsFun: inputs: sourceInfo: let outputs = outputsFun inputs; in " + "outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }", "/"), vCall); - state.callFunction(v2, v, vRes, noPos); + auto vCall2 = *state.allocValue(); + auto vCall3 = *state.allocValue(); + state.callFunction(vCall, *flake.vOutputs, vCall2, noPos); + state.callFunction(vCall2, vInputs, vCall3, noPos); + state.callFunction(vCall3, vSourceInfo, vRes, noPos); } void callFlake(EvalState & state, From d749f5132b7da9676e5dec2b1ef596024a23da95 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 13:06:48 +0200 Subject: [PATCH 0454/2548] Update flake.{nix,lock} --- flake.lock | 4 ++-- flake.nix | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 336e3eb86..8072152ca 100644 --- a/flake.lock +++ b/flake.lock @@ -3,9 +3,9 @@ "nixpkgs": { "id": "nixpkgs", "inputs": {}, - "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", + "narHash": "sha256-AndIaZrFFIT+VFhVtQHsS90I5SWfjTDTxzs9Hx9ZxZA=", "nonFlakeInputs": {}, - "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" + "uri": "github:edolstra/nixpkgs/9bd2e2c96ddeec64e9ad37540412263fdb78458d" } }, "nonFlakeInputs": {}, diff --git a/flake.nix b/flake.nix index e7a85d730..9f7d6b2ee 100644 --- a/flake.nix +++ b/flake.nix @@ -3,15 +3,13 @@ description = "The purely functional package manager"; - edition = 201906; + edition = 201909; - inputs = [ "nixpkgs" ]; - - outputs = inputs: rec { + outputs = { self, nixpkgs }: rec { hydraJobs = import ./release.nix { - nix = inputs.self; - nixpkgs = inputs.nixpkgs; + nix = self; + nixpkgs = nixpkgs; }; checks = { @@ -29,7 +27,7 @@ defaultPackage = packages.nix; devShell = import ./shell.nix { - nixpkgs = inputs.nixpkgs; + inherit nixpkgs; }; }; } From 0588d72286c97c7b43d8682930e0e43b0a1b8c1a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 13:11:33 +0200 Subject: [PATCH 0455/2548] Update tests --- tests/flakes.sh | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 406e5b632..8bb6e90a2 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -31,7 +31,7 @@ cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix <&1 | grep 'unsupported edition' # Test whether registry caching works. @@ -244,7 +238,7 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Fri, 30 Aug 2019 16:27:51 +0200 Subject: [PATCH 0456/2548] Turn flake inputs into an attrset Instead of a list, inputs are now an attrset like inputs = { nixpkgs.uri = github:NixOS/nixpkgs; }; If 'uri' is omitted, than the flake is a lookup in the flake registry, e.g. inputs = { nixpkgs = {}; }; but in that case, you can also just omit the input altogether and specify it as an argument to the 'outputs' function, as in outputs = { self, nixpkgs }: ... This also gets rid of 'nonFlakeInputs', which are now just a special kind of input that have a 'flake = false' attribute, e.g. inputs = { someRepo = { uri = github:example/repo; flake = false; }; }; --- src/libexpr/flake/flake.cc | 167 ++++++++++++++------------------ src/libexpr/flake/flake.hh | 20 ++-- src/libexpr/flake/lockfile.cc | 52 +++------- src/libexpr/flake/lockfile.hh | 79 +++++---------- src/libexpr/primops/fetchGit.cc | 1 - src/nix/installables.cc | 11 +-- tests/flakes.sh | 26 ++--- 7 files changed, 136 insertions(+), 220 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2a6792af9..ce5dd55a2 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -248,22 +248,28 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); auto sInputs = state.symbols.create("inputs"); + auto sUri = state.symbols.create("uri"); + auto sFlake = state.symbols.create("flake"); - if (auto inputs = vInfo.attrs->get(sInputs)) { - state.forceList(*(**inputs).value, *(**inputs).pos); - for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) - flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( - *(**inputs).value->listElems()[n], *(**inputs).pos))); - } + if (std::optional inputs = vInfo.attrs->get(sInputs)) { + state.forceAttrs(*(**inputs).value, *(**inputs).pos); - auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); + for (Attr inputAttr : *(*(**inputs).value).attrs) { + state.forceAttrs(*inputAttr.value, *inputAttr.pos); - if (std::optional nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { - state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); - for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { - std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); - FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); - flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); + FlakeInput input(FlakeRef(inputAttr.name)); + + for (Attr attr : *(inputAttr.value->attrs)) { + if (attr.name == sUri) { + input.ref = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (attr.name == sFlake) { + input.isFlake = state.forceBool(*attr.value, *attr.pos); + } else + throw Error("flake input '%s' has an unsupported attribute '%s', at %s", + inputAttr.name, attr.name, *attr.pos); + } + + flake.inputs.emplace(inputAttr.name, input); } } @@ -275,9 +281,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (flake.vOutputs->lambda.fun->matchAttrs) { for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) { - if (formal.name != state.sSelf) { - flake.inputs.push_back(FlakeRef(formal.name)); - } + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput(FlakeRef(formal.name))); } } @@ -290,7 +295,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) attr.name != state.sName && attr.name != state.sDescription && attr.name != sInputs && - attr.name != sNonFlakeInputs && attr.name != sOutputs) throw Error("flake '%s' has an unsupported attribute '%s', at %s", flakeRef, attr.name, *attr.pos); @@ -299,21 +303,19 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) return flake; } -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef) +static SourceInfo getNonFlake(EvalState & state, const FlakeRef & flakeRef) { auto sourceInfo = fetchFlake(state, flakeRef); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; - NonFlake nonFlake(flakeRef, sourceInfo); - - state.store->assertStorePath(nonFlake.sourceInfo.storePath); + state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) - state.allowedPaths->insert(nonFlake.sourceInfo.storePath); + state.allowedPaths->insert(sourceInfo.storePath); - return nonFlake; + return sourceInfo; } bool allowedToWrite(HandleLockFile handle) @@ -346,46 +348,33 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) Note that this is lazy: we only recursively fetch inputs that are not in the lockfile yet. */ -static std::pair updateLocks( +static std::pair updateLocks( EvalState & state, const Flake & flake, HandleLockFile handleLockFile, - const FlakeInputs & oldEntry, + const LockedInputs & oldEntry, bool topRef) { - FlakeInput newEntry( - flake.id, + LockedInput newEntry( flake.sourceInfo.resolvedRef, flake.sourceInfo.narHash); - for (auto & input : flake.nonFlakeInputs) { - auto & id = input.first; - auto & ref = input.second; - auto i = oldEntry.nonFlakeInputs.find(id); - if (i != oldEntry.nonFlakeInputs.end()) { - newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); + for (auto & [id, input] : flake.inputs) { + auto i = oldEntry.inputs.find(id); + if (i != oldEntry.inputs.end()) { + newEntry.inputs.insert_or_assign(id, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, maybeLookupFlake(state, ref, allowedToUseRegistries(handleLockFile, false))); - newEntry.nonFlakeInputs.insert_or_assign(id, - NonFlakeInput( - nonFlake.sourceInfo.resolvedRef, - nonFlake.sourceInfo.narHash)); - } - } - - for (auto & inputRef : flake.inputs) { - auto i = oldEntry.flakeInputs.find(inputRef); - if (i != oldEntry.flakeInputs.end()) { - newEntry.flakeInputs.insert_or_assign(inputRef, i->second); - } else { - if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update flake dependency '%s' in pure mode", inputRef); - newEntry.flakeInputs.insert_or_assign(inputRef, - updateLocks(state, - getFlake(state, maybeLookupFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false))), - handleLockFile, {}, false).second); + throw Error("cannot update flake input '%s' in pure mode", id); + if (input.isFlake) + newEntry.inputs.insert_or_assign(id, + updateLocks(state, + getFlake(state, maybeLookupFlake(state, input.ref, allowedToUseRegistries(handleLockFile, false))), + handleLockFile, {}, false).second); + else { + auto sourceInfo = getNonFlake(state, maybeLookupFlake(state, input.ref, allowedToUseRegistries(handleLockFile, false))); + newEntry.inputs.insert_or_assign(id, LockedInput(sourceInfo.resolvedRef, sourceInfo.narHash)); + } } } @@ -462,81 +451,69 @@ static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); } +struct LazyInput +{ + bool isFlake; + LockedInput lockedInput; +}; + /* Helper primop to make callFlake (below) fetch/call its inputs lazily. Note that this primop cannot be called by user code since it doesn't appear in 'builtins'. */ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - auto lazyFlake = (FlakeInput *) args[0]->attrs; + auto lazyInput = (LazyInput *) args[0]->attrs; - assert(lazyFlake->ref.isImmutable()); + assert(lazyInput->lockedInput.ref.isImmutable()); - auto flake = getFlake(state, lazyFlake->ref); + if (lazyInput->isFlake) { + auto flake = getFlake(state, lazyInput->lockedInput.ref); - if (flake.sourceInfo.narHash != lazyFlake->narHash) - throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + if (flake.sourceInfo.narHash != lazyInput->lockedInput.narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); - callFlake(state, flake, *lazyFlake, v); -} + callFlake(state, flake, lazyInput->lockedInput, v); + } else { + auto sourceInfo = getNonFlake(state, lazyInput->lockedInput.ref); -static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + if (sourceInfo.narHash != lazyInput->lockedInput.narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", sourceInfo.resolvedRef); - assert(lazyNonFlake->ref.isImmutable()); + state.mkAttrs(v, 8); - auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + assert(state.store->isValidPath(sourceInfo.storePath)); - if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) - throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + mkString(*state.allocAttr(v, state.sOutPath), + sourceInfo.storePath, {sourceInfo.storePath}); - state.mkAttrs(v, 8); - - assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); - - mkString(*state.allocAttr(v, state.sOutPath), - nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - - emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); + emitSourceInfoAttrs(state, sourceInfo, v); + } } void callFlake(EvalState & state, const Flake & flake, - const FlakeInputs & inputs, + const LockedInputs & lockedInputs, Value & vRes) { auto & vInputs = *state.allocValue(); - state.mkAttrs(vInputs, - inputs.flakeInputs.size() + - inputs.nonFlakeInputs.size() + 1); + state.mkAttrs(vInputs, flake.inputs.size() + 1); - for (auto & dep : inputs.flakeInputs) { - auto vFlake = state.allocAttr(vInputs, dep.second.id); + for (auto & [inputId, input] : flake.inputs) { + auto vFlake = state.allocAttr(vInputs, inputId); auto vPrimOp = state.allocValue(); static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); vPrimOp->type = tPrimOp; vPrimOp->primOp = primOp; auto vArg = state.allocValue(); vArg->type = tNull; + auto lockedInput = lockedInputs.inputs.find(inputId); + assert(lockedInput != lockedInputs.inputs.end()); // FIXME: leak - vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient + vArg->attrs = (Bindings *) new LazyInput{input.isFlake, lockedInput->second}; mkApp(*vFlake, *vPrimOp, *vArg); } - for (auto & dep : inputs.nonFlakeInputs) { - auto vNonFlake = state.allocAttr(vInputs, dep.first); - auto vPrimOp = state.allocValue(); - static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = primOp; - auto vArg = state.allocValue(); - vArg->type = tNull; - // FIXME: leak - vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient - mkApp(*vNonFlake, *vPrimOp, *vArg); - } - auto & vSourceInfo = *state.allocValue(); state.mkAttrs(vSourceInfo, 8); emitSourceInfoAttrs(state, flake.sourceInfo, vSourceInfo); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 01fb421bd..84bce9b5a 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -58,14 +58,20 @@ struct SourceInfo SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; }; +struct FlakeInput +{ + FlakeRef ref; + bool isFlake = true; + FlakeInput(const FlakeRef & ref) : ref(ref) {}; +}; + struct Flake { FlakeId id; FlakeRef originalRef; std::string description; SourceInfo sourceInfo; - std::vector inputs; - std::map nonFlakeInputs; + std::map inputs; Value * vOutputs; // FIXME: gc unsigned int edition; @@ -73,14 +79,6 @@ struct Flake : originalRef(origRef), sourceInfo(sourceInfo) {}; }; -struct NonFlake -{ - FlakeRef originalRef; - SourceInfo sourceInfo; - NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) - : originalRef(origRef), sourceInfo(sourceInfo) {}; -}; - Flake getFlake(EvalState &, const FlakeRef &); /* If 'allowLookup' is true, then resolve 'flakeRef' using the @@ -108,7 +106,7 @@ ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); void callFlake(EvalState & state, const Flake & flake, - const FlakeInputs & inputs, + const LockedInputs & inputs, Value & v); void callFlake(EvalState & state, diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 15f2e2e8e..f32d752f9 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -3,83 +3,57 @@ namespace nix::flake { -AbstractInput::AbstractInput(const nlohmann::json & json) - : ref(json["uri"]) +LockedInput::LockedInput(const nlohmann::json & json) + : LockedInputs(json) + , ref(json["uri"]) , narHash(Hash((std::string) json["narHash"])) { if (!ref.isImmutable()) throw Error("lockfile contains mutable flakeref '%s'", ref); } -nlohmann::json AbstractInput::toJson() const +nlohmann::json LockedInput::toJson() const { - nlohmann::json json; + auto json = LockedInputs::toJson(); json["uri"] = ref.to_string(); json["narHash"] = narHash.to_string(SRI); return json; } -Path AbstractInput::computeStorePath(Store & store) const +Path LockedInput::computeStorePath(Store & store) const { return store.makeFixedOutputPath(true, narHash, "source"); } -FlakeInput::FlakeInput(const nlohmann::json & json) - : FlakeInputs(json) - , AbstractInput(json) - , id(json["id"]) +LockedInputs::LockedInputs(const nlohmann::json & json) { -} - -nlohmann::json FlakeInput::toJson() const -{ - auto json = FlakeInputs::toJson(); - json.update(AbstractInput::toJson()); - json["id"] = id; - return json; -} - -FlakeInputs::FlakeInputs(const nlohmann::json & json) -{ - for (auto & i : json["nonFlakeInputs"].items()) - nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); - for (auto & i : json["inputs"].items()) - flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); + inputs.insert_or_assign(i.key(), LockedInput(i.value())); } -nlohmann::json FlakeInputs::toJson() const +nlohmann::json LockedInputs::toJson() const { nlohmann::json json; { auto j = nlohmann::json::object(); - for (auto & i : nonFlakeInputs) + for (auto & i : inputs) j[i.first] = i.second.toJson(); - json["nonFlakeInputs"] = std::move(j); - } - { - auto j = nlohmann::json::object(); - for (auto & i : flakeInputs) - j[i.first.to_string()] = i.second.toJson(); json["inputs"] = std::move(j); } return json; } -bool FlakeInputs::isDirty() const +bool LockedInputs::isDirty() const { - for (auto & i : flakeInputs) + for (auto & i : inputs) if (i.second.ref.isDirty() || i.second.isDirty()) return true; - for (auto & i : nonFlakeInputs) - if (i.second.ref.isDirty()) return true; - return false; } nlohmann::json LockFile::toJson() const { - auto json = FlakeInputs::toJson(); + auto json = LockedInputs::toJson(); json["version"] = 2; return json; } diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 7077db3cd..084eabc1a 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -10,47 +10,15 @@ class Store; namespace nix::flake { -/* Common lock file information about a flake input, namely the - immutable ref and the NAR hash. */ -struct AbstractInput -{ - FlakeRef ref; - Hash narHash; - - AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) - : ref(flakeRef), narHash(narHash) - { - assert(ref.isImmutable()); - }; - - AbstractInput(const nlohmann::json & json); - - nlohmann::json toJson() const; - - Path computeStorePath(Store & store) const; -}; - -/* Lock file information about a non-flake input. */ -struct NonFlakeInput : AbstractInput -{ - using AbstractInput::AbstractInput; - - bool operator ==(const NonFlakeInput & other) const - { - return ref == other.ref && narHash == other.narHash; - } -}; - -struct FlakeInput; +struct LockedInput; /* Lock file information about the dependencies of a flake. */ -struct FlakeInputs +struct LockedInputs { - std::map flakeInputs; - std::map nonFlakeInputs; + std::map inputs; - FlakeInputs() {}; - FlakeInputs(const nlohmann::json & json); + LockedInputs() {}; + LockedInputs(const nlohmann::json & json); nlohmann::json toJson() const; @@ -60,47 +28,48 @@ struct FlakeInputs }; /* Lock file information about a flake input. */ -struct FlakeInput : FlakeInputs, AbstractInput +struct LockedInput : LockedInputs { - FlakeId id; + FlakeRef ref; + Hash narHash; - FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractInput(flakeRef, narHash), id(id) {}; + LockedInput(const FlakeRef & ref, const Hash & narHash) + : ref(ref), narHash(narHash) + { + assert(ref.isImmutable()); + }; - FlakeInput(const nlohmann::json & json); + LockedInput(const nlohmann::json & json); - bool operator ==(const FlakeInput & other) const + bool operator ==(const LockedInput & other) const { return - id == other.id - && ref == other.ref + ref == other.ref && narHash == other.narHash - && flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; + && inputs == other.inputs; } nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; }; /* An entire lock file. Note that this cannot be a FlakeInput for the top-level flake, because then the lock file would need to contain the hash of the top-level flake, but committing the lock file would invalidate that hash. */ -struct LockFile : FlakeInputs +struct LockFile : LockedInputs { bool operator ==(const LockFile & other) const { - return - flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; + return inputs == other.inputs; } LockFile() {} - LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeInput && dep) + LockFile(const nlohmann::json & json) : LockedInputs(json) {} + LockFile(LockedInput && dep) { - flakeInputs = std::move(dep.flakeInputs); - nonFlakeInputs = std::move(dep.nonFlakeInputs); + inputs = std::move(dep.inputs); } nlohmann::json toJson() const; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 940cf3022..931eac95f 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -31,7 +31,6 @@ GitInfo exportGit(ref store, std::string uri, // or revision is given, then allow the use of an unclean working // tree. if (!ref && !rev && isLocal) { - bool clean = true; try { diff --git a/src/nix/installables.cc b/src/nix/installables.cc index aa5ef5184..dbbf58861 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -217,25 +217,20 @@ void makeFlakeClosureGCRoot(Store & store, assert(store.isValidPath(resFlake.flake.sourceInfo.storePath)); closure.insert(resFlake.flake.sourceInfo.storePath); - std::queue> queue; + std::queue> queue; queue.push(resFlake.lockFile); while (!queue.empty()) { - const flake::FlakeInputs & flake = queue.front(); + const flake::LockedInputs & flake = queue.front(); queue.pop(); /* Note: due to lazy fetching, these paths might not exist yet. */ - for (auto & dep : flake.flakeInputs) { + for (auto & dep : flake.inputs) { auto path = dep.second.computeStorePath(store); if (store.isValidPath(path)) closure.insert(path); queue.push(dep.second); } - for (auto & dep : flake.nonFlakeInputs) { - auto path = dep.second.computeStorePath(store); - if (store.isValidPath(path)) - closure.insert(path); - } } if (closure.empty()) return; diff --git a/tests/flakes.sh b/tests/flakes.sh index 8bb6e90a2..7c16304ec 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -240,10 +240,13 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out + cat \${nonFlake}/README.md > \$out ''; }; }; From 80c36d4562af71a90c67b3adb886a1003834890e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 16:38:27 +0200 Subject: [PATCH 0457/2548] Remove 'name' attribute from flakes This is no longer needed since flakes are given an identity in the 'inputs' attribute. --- flake.lock | 7 ++----- flake.nix | 2 -- src/libexpr/flake/flake.cc | 6 ------ src/libexpr/flake/flake.hh | 1 - src/nix/flake.cc | 11 ----------- tests/flakes.sh | 20 ++++---------------- 6 files changed, 6 insertions(+), 41 deletions(-) diff --git a/flake.lock b/flake.lock index 8072152ca..fa5649c03 100644 --- a/flake.lock +++ b/flake.lock @@ -1,13 +1,10 @@ { "inputs": { "nixpkgs": { - "id": "nixpkgs", "inputs": {}, - "narHash": "sha256-AndIaZrFFIT+VFhVtQHsS90I5SWfjTDTxzs9Hx9ZxZA=", - "nonFlakeInputs": {}, - "uri": "github:edolstra/nixpkgs/9bd2e2c96ddeec64e9ad37540412263fdb78458d" + "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", + "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" } }, - "nonFlakeInputs": {}, "version": 2 } diff --git a/flake.nix b/flake.nix index 9f7d6b2ee..06a5515c5 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,4 @@ { - name = "nix"; - description = "The purely functional package manager"; edition = 201909; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ce5dd55a2..ba337cf97 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -239,11 +239,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake '%s' lacks attribute 'edition'", flakeRef); - if (auto name = vInfo.attrs->get(state.sName)) - flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); - else - throw Error("flake '%s' lacks attribute 'name'", flakeRef); - if (auto description = vInfo.attrs->get(state.sDescription)) flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); @@ -292,7 +287,6 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) for (auto & attr : *vInfo.attrs) { if (attr.name != sEdition && attr.name != sEpoch && - attr.name != state.sName && attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs) diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 84bce9b5a..8f05e9799 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -67,7 +67,6 @@ struct FlakeInput struct Flake { - FlakeId id; FlakeRef originalRef; std::string description; SourceInfo sourceInfo; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index aab29b626..4129ef323 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -103,7 +103,6 @@ static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) static void printFlakeInfo(const Flake & flake) { - std::cout << fmt("ID: %s\n", flake.id); std::cout << fmt("Description: %s\n", flake.description); std::cout << fmt("Edition: %s\n", flake.edition); printSourceInfo(flake.sourceInfo); @@ -112,7 +111,6 @@ static void printFlakeInfo(const Flake & flake) static nlohmann::json flakeToJson(const Flake & flake) { nlohmann::json j; - j["id"] = flake.id; j["description"] = flake.description; j["edition"] = flake.edition; sourceInfoToJson(flake.sourceInfo, j); @@ -120,12 +118,6 @@ static nlohmann::json flakeToJson(const Flake & flake) } #if 0 -static void printNonFlakeInfo(const NonFlake & nonFlake) -{ - std::cout << fmt("ID: %s\n", nonFlake.alias); - printSourceInfo(nonFlake.sourceInfo); -} - // FIXME: merge info CmdFlakeInfo? struct CmdFlakeDeps : FlakeCommand { @@ -148,9 +140,6 @@ struct CmdFlakeDeps : FlakeCommand auto resFlake = std::move(todo.front()); todo.pop(); - for (auto & nonFlake : resFlake.nonFlakeDeps) - printNonFlakeInfo(nonFlake); - for (auto & info : resFlake.flakeDeps) { printFlakeInfo(info.second.flake); todo.push(info.second); diff --git a/tests/flakes.sh b/tests/flakes.sh index 7c16304ec..8594a4c55 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -29,8 +29,6 @@ done cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Fri, 30 Aug 2019 17:27:51 +0200 Subject: [PATCH 0458/2548] Fix sourceInfo --- src/libexpr/flake/flake.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ba337cf97..9e25fc116 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -511,6 +511,7 @@ void callFlake(EvalState & state, auto & vSourceInfo = *state.allocValue(); state.mkAttrs(vSourceInfo, 8); emitSourceInfoAttrs(state, flake.sourceInfo, vSourceInfo); + vSourceInfo.attrs->sort(); vInputs.attrs->push_back(Attr(state.sSelf, &vRes)); From 5ec2a1ed82d485429aaf6fbad55fd6c1320b2d8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Sep 2019 15:59:19 +0200 Subject: [PATCH 0459/2548] nix dev-shell --profile: Support relative path --- src/nix/command.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix/command.cc b/src/nix/command.cc index 8191cb831..9cca443dc 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -96,10 +96,11 @@ void MixProfile::updateProfile(const Path & storePath) if (!profile) return; auto store = getStore().dynamic_pointer_cast(); if (!store) throw Error("'--profile' is not supported for this Nix store"); - switchLink(*profile, + auto profile2 = absPath(*profile); + switchLink(profile2, createGeneration( ref(store), - *profile, storePath)); + profile2, storePath)); } void MixProfile::updateProfile(const Buildables & buildables) From 61fdb16aacf9ff18c96b72a37e1b46eb14586eb4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Sep 2019 17:33:07 +0200 Subject: [PATCH 0460/2548] Improve error message when a directory is not a flake So you now get $ nix build error: path '.' is not a flake (because it does not reference a Git repository) rather than $ nix build error: unsupported argument '.' --- src/libexpr/flake/flakeref.cc | 6 ++-- src/libexpr/flake/flakeref.hh | 1 + src/nix/installables.cc | 52 +++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 7631cd53a..253442566 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -145,10 +145,10 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) d.path = absPath(uri); try { if (!S_ISDIR(lstat(d.path).st_mode)) - throw BadFlakeRef("path '%s' is not a flake (sub)directory"); + throw MissingFlake("path '%s' is not a flake (sub)directory", d.path); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == EISDIR) - throw BadFlakeRef("flake '%s' does not exist"); + throw MissingFlake("flake '%s' does not exist", d.path); throw; } while (true) { @@ -156,7 +156,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); d.path = dirOf(d.path); if (d.path == "/") - throw BadFlakeRef("path '%s' does not reference a Git repository", uri); + throw MissingFlake("path '%s' is not a flake (because it does not reference a Git repository)", uri); } } else d.path = canonPath(uri); diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 082dd8c26..9ddc227bb 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -187,6 +187,7 @@ struct FlakeRef std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); MakeError(BadFlakeRef, Error); +MakeError(MissingFlake, BadFlakeRef); std::optional parseFlakeRef( const std::string & uri, bool allowRelative = false); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index dbbf58861..a4726a59e 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -417,25 +417,43 @@ std::vector> SourceExprCommand::parseInstallables( Strings{"legacyPackages." + std::string(s, 8)})); } - else if (auto flakeRef = parseFlakeRef(s, true)) - result.push_back(std::make_shared(*this, std::move(*flakeRef), - getDefaultFlakeAttrPaths())); + else { - else if ((colon = s.rfind(':')) != std::string::npos) { - auto flakeRef = std::string(s, 0, colon); - auto attrPath = std::string(s, colon + 1); - result.push_back(std::make_shared( - *this, - FlakeRef(flakeRef, true), - attrPath, - getDefaultFlakeAttrPathPrefixes())); + std::exception_ptr flakeEx; + + try { + auto flakeRef = FlakeRef(s, true); + result.push_back(std::make_shared( + *this, std::move(flakeRef), getDefaultFlakeAttrPaths())); + continue; + } catch (MissingFlake &) { + /* 's' could be parsed as a flakeref, but it + references a local path that is not a flake. So + take note of that. */ + flakeEx = std::current_exception(); + } catch (BadFlakeRef &) { + } + + if ((colon = s.rfind(':')) != std::string::npos) { + auto flakeRef = std::string(s, 0, colon); + auto attrPath = std::string(s, colon + 1); + result.push_back(std::make_shared( + *this, + FlakeRef(flakeRef, true), + attrPath, + getDefaultFlakeAttrPathPrefixes())); + } + + else if (s.find('/') != std::string::npos && (storePath = follow(s))) + result.push_back(std::make_shared(*storePath)); + + else { + if (flakeEx) + std::rethrow_exception(flakeEx); + else + throw Error("unsupported argument '%s'", s); + } } - - else if (s.find('/') != std::string::npos && (storePath = follow(s))) - result.push_back(std::make_shared(*storePath)); - - else - throw Error("unsupported argument '%s'", s); } } From c693f80b814c244dcdae7a2e87fb9e444d9d1ca5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Sep 2019 17:43:27 +0200 Subject: [PATCH 0461/2548] Shut up some clang warnings --- src/libutil/args.hh | 2 ++ src/nix/command.hh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index a083c4ce8..b960a55a8 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -199,6 +199,8 @@ private: public: + virtual ~Command() { } + std::string name() { return _name; } virtual void prepare() { }; diff --git a/src/nix/command.hh b/src/nix/command.hh index 00c202f20..92f606bbe 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -49,6 +49,8 @@ struct App struct Installable { + virtual ~Installable() { } + virtual std::string what() = 0; virtual Buildables toBuildables() From aeb695c0074b52772057b36f442a054f8d1a856d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Sep 2019 17:50:44 +0200 Subject: [PATCH 0462/2548] Remove obsolete OpenSSL locking code OpenSSL 1.1.1 no longer needs this (https://github.com/openssl/openssl/commit/2e52e7df518d80188c865ea3f7bb3526d14b0c08). This shuts up a clang warning about opensslLockCallback being unused. --- src/libmain/shared.cc | 20 -------------------- src/libutil/hash.cc | 4 ++++ 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0afddfb78..910549583 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,8 +15,6 @@ #include #include -#include - namespace nix { @@ -80,20 +78,6 @@ string getArg(const string & opt, } -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} - - static void sigHandler(int signo) { } @@ -105,10 +89,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); - loadConfFile(); startSignalHandlerThread(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1c14ebb18..362c537fe 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -13,6 +13,10 @@ #include #include +#if OPENSSL_VERSION_NUMBER < 0x10101000L +#error "Unsupported version of OpenSSL, you need at least 1.1.1" +#endif + namespace nix { From 2dbd69dbf4538d5b7947d192979ff4feab322c2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Sep 2019 23:04:27 +0200 Subject: [PATCH 0463/2548] nix repl: Run in impure mode --- src/nix/repl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 3a70a23c7..0fa1594cc 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -778,6 +778,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref store) override { + evalSettings.pureEval = false; auto repl = std::make_unique(searchPath, openStore()); repl->autoArgs = getAutoArgs(repl->state); repl->mainLoop(files); From 87c604c1f04ff31a68f2fa73fe5a2ed9f6ec7ac7 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 2 Sep 2019 18:35:10 -0400 Subject: [PATCH 0464/2548] Fix launchd program args launchd has some weird syntx. Apparently the program needs to be in the ProgramArguments, as Program appears to be ignored. --- misc/launchd/org.nixos.nix-daemon.plist.in | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 6bc448d28..b340610e9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -13,10 +13,9 @@ RunAtLoad - Program - /bin/sh ProgramArguments + /bin/sh -c /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon From e4ea3e03066a760c8cd462108af99aebaaa44c1b Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Tue, 3 Sep 2019 07:32:44 +0200 Subject: [PATCH 0465/2548] docs: Note that tryEval doesn't do deep evaluation --- doc/manual/expressions/builtins.xml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4c1d618e9..f73d2e1ae 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -1607,12 +1607,18 @@ stdenv.mkDerivation (rec { builtins.tryEval e - Try to evaluate e. + Try to shallowly evaluate e. Return a set containing the attributes success (true if e evaluated successfully, false if an error was thrown) and value, equalling e - if successful and false otherwise. + if successful and false otherwise. Note that this + doesn't evaluate e deeply, so + let e = { x = throw ""; }; in (builtins.tryEval e).success + will be true. Using builtins.deepSeq + one can get the expected result: let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success will be + false. From 8c4ea7a4516c517a0dd37b446bf5c1a6b157064c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 12:51:14 +0200 Subject: [PATCH 0466/2548] Downloader: Remove a possible double call to Callback --- src/libstore/download.cc | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index c322d267d..a7d059465 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -342,15 +342,9 @@ struct CurlDownloader : public Downloader (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); done = true; - - try { - act.progress(result.bodySize, result.bodySize); - callback(std::move(result)); - } catch (...) { - done = true; - callback.rethrow(); - } + callback(std::move(result)); } else { From 7348653ff4fc4e9b2dc24943aabdb57179b1c75a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 12:51:35 +0200 Subject: [PATCH 0467/2548] Ensure that Callback is called only once Also, make Callback movable but uncopyable. --- src/libstore/binary-cache-store.cc | 8 +++++--- src/libstore/download.cc | 6 +++--- src/libstore/http-binary-cache-store.cc | 12 +++++++----- src/libstore/store-api.cc | 8 +++++--- src/libutil/util.hh | 19 ++++++++++++++++--- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..e56be625d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -249,21 +249,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared(std::move(callback)); + getFile(narInfoFile, {[=](std::future> fut) { try { auto data = fut.get(); - if (!data) return callback(nullptr); + if (!data) return (*callbackPtr)(nullptr); stats.narInfoRead++; - callback((std::shared_ptr) + (*callbackPtr)((std::shared_ptr) std::make_shared(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/download.cc b/src/libstore/download.cc index a7d059465..cdf56e09d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -77,13 +77,13 @@ struct CurlDownloader : public Downloader DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, - Callback callback) + Callback && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) - , callback(callback) + , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { writtenToSink += len; @@ -665,7 +665,7 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared(*this, request, callback)); + enqueueItem(std::make_shared(*this, request, std::move(callback))); } }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index df2fb9332..e631d95f0 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -137,17 +137,19 @@ protected: auto request(makeRequest(path)); + auto callbackPtr = std::make_shared(std::move(callback)); + getDownloader()->enqueueDownload(request, - {[callback, this](std::future result) { + {[callbackPtr, this](std::future result) { try { - callback(result.get().data); + (*callbackPtr)(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr()); + return (*callbackPtr)(std::shared_ptr()); maybeDisable(); - callback.rethrow(); + callbackPtr->rethrow(); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3bb9db0b7..88a5b2f44 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -365,8 +365,10 @@ void Store::queryPathInfo(const Path & storePath, } catch (...) { return callback.rethrow(); } + auto callbackPtr = std::make_shared(std::move(callback)); + queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callback](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -386,8 +388,8 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - callback(ref(info)); - } catch (...) { callback.rethrow(); } + (*callbackPtr)(ref(info)); + } catch (...) { callbackPtr->rethrow(); } }}); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b538a0b41..686e81d3f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -445,21 +445,34 @@ string get(const T & map, const string & key, const string & def = "") type T or an exception. (We abuse std::future to pass the value or exception.) */ template -struct Callback +class Callback { std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: Callback(std::function)> fun) : fun(fun) { } - void operator()(T && t) const + Callback(Callback && callback) : fun(std::move(callback.fun)) { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) + { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_value(std::move(t)); fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) const + void rethrow(const std::exception_ptr & exc = std::current_exception()) { + auto prev = done.test_and_set(); + assert(!prev); std::promise promise; promise.set_exception(exc); fun(promise.get_future()); From f186000367978fbe590343f47951232f42a30bec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 13:00:55 +0200 Subject: [PATCH 0468/2548] Add some noexcepts This is to assert that callback functions should never throw (since the context in which they're called may not be able to handle the exception). --- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/binary-cache-store.hh | 4 ++-- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.cc | 9 +++++---- src/libstore/store-api.hh | 4 ++-- src/libutil/util.hh | 4 ++-- 11 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e56be625d..10cde8704 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -55,7 +55,7 @@ void BinaryCacheStore::init() } void BinaryCacheStore::getFile(const std::string & path, - Callback> callback) + Callback> callback) noexcept { try { callback(getFile(path)); @@ -240,7 +240,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { auto uri = getUri(); auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 953f3b90a..af108880c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -47,7 +47,7 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - Callback> callback); + Callback> callback) noexcept; std::shared_ptr getFile(const std::string & path); @@ -73,7 +73,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index e631d95f0..779f89e68 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -131,7 +131,7 @@ protected: } void getFile(const std::string & path, - Callback> callback) override + Callback> callback) noexcept override { checkEnabled(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7c9bc2b68..d5fbdd25a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback> callback) override + Callback> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63b11467e..2fcf08491 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -629,7 +629,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { auto info = std::make_shared(); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index af8b84bf5..3ae34c403 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c2e23f9c..e38fe49a7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -349,7 +349,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback> callback) + Callback> callback) noexcept { try { std::shared_ptr info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80f18ab71..82fbec092 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -41,7 +41,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback> callback) override; + Callback> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 88a5b2f44..5f63c53b5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -329,13 +329,14 @@ ref Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback> callback) + Callback> callback) noexcept { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); { auto res = state.lock()->pathInfoCache.get(hashPart); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 599677376..7fb568602 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,12 +360,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback> callback); + Callback> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback> callback) = 0; + Callback> callback) noexcept = 0; public: diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 686e81d3f..07c3d28ff 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -460,7 +460,7 @@ public: if (prev) done.test_and_set(); } - void operator()(T && t) + void operator()(T && t) noexcept { auto prev = done.test_and_set(); assert(!prev); @@ -469,7 +469,7 @@ public: fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept { auto prev = done.test_and_set(); assert(!prev); From cec50290bf5537d574fd94557bd3918f2bea8a30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 15:45:32 +0200 Subject: [PATCH 0469/2548] gc-auto.sh: Add some more instrumentation --- tests/gc-auto.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 54ead227e..5c8be7c0e 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -17,6 +17,7 @@ expr=$(cat < \$out/bar @@ -32,14 +33,11 @@ with import ./config.nix; mkDerivation { EOF ) -nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ - --min-free 1000 --max-free 2000 --min-free-check-interval 1 & -pid=$! - expr2=$(cat < \$out/bar echo 1... @@ -53,6 +51,10 @@ with import ./config.nix; mkDerivation { EOF ) +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 From e07ec8d27e08bf23eccab079b044a6f1b37f3ac9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 16:02:12 +0200 Subject: [PATCH 0470/2548] Support allowSubstitutes attribute in structured attribute derivations Hopefully fixes #3081 (didn't test). --- src/libstore/build.cc | 2 +- src/libstore/derivations.cc | 6 ------ src/libstore/derivations.hh | 2 -- src/libstore/misc.cc | 4 +++- src/libstore/parsed-derivations.cc | 5 +++++ src/libstore/parsed-derivations.hh | 2 ++ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index be52b66a7..ab725e8e9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1197,7 +1197,7 @@ void DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3961126ff..23fcfb281 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::substitutesAllowed() const -{ - return get(env, "allowSubstitutes", "1") == "1"; -} - - bool BasicDerivation::isBuiltin() const { return string(builder, 0, 8) == "builtin:"; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9753e796d..8e02c9bc5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,8 +56,6 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool substitutesAllowed() const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index adcce026f..dddf13430 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets, } Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); PathSet invalid; for (auto & j : drv.outputs) @@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets, invalid.insert(j.second.path); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, i2.first, make_ref(drv), output, drvState)); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 17fde00a0..87be8a24e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const return getBoolAttr("preferLocalBuild") && canBuildLocally(); } +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index ed07dc652..9bde4b4dc 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -30,6 +30,8 @@ public: bool canBuildLocally() const; bool willBuildLocally() const; + + bool substitutesAllowed() const; }; } From 08ee36495087e3d6f573615c050497ec2298cafa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Sep 2019 18:11:43 +0200 Subject: [PATCH 0471/2548] gc-auto.sh: More test fixes --- tests/gc-auto.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 5c8be7c0e..87f1d6798 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -23,10 +23,13 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 200 > $fake_free + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free echo 2... - sleep 2 + sleep 1 echo 3... + sleep 1 + echo 4... [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] ''; } @@ -42,10 +45,13 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 200 > $fake_free + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free echo 2... - sleep 2 + sleep 1 echo 3... + sleep 1 + echo 4... ''; } EOF From 5fad9d01c242ffa9d83cddeeacb9abbaa3848c7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 12:52:54 +0200 Subject: [PATCH 0472/2548] gc-auto.sh: Increase sleep time --- tests/gc-auto.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index 87f1d6798..de1e2cfe4 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -26,9 +26,9 @@ with import ./config.nix; mkDerivation { echo 200 > ${fake_free}.tmp1 mv ${fake_free}.tmp1 $fake_free echo 2... - sleep 1 + sleep 2 echo 3... - sleep 1 + sleep 2 echo 4... [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] ''; @@ -48,9 +48,9 @@ with import ./config.nix; mkDerivation { echo 200 > ${fake_free}.tmp2 mv ${fake_free}.tmp2 $fake_free echo 2... - sleep 1 + sleep 2 echo 3... - sleep 1 + sleep 2 echo 4... ''; } From b774845af7a645b44bff69cf9f655c47fe4b9fb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 12:53:22 +0200 Subject: [PATCH 0473/2548] Set release date --- doc/manual/release-notes/rl-2.3.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index a1f4a4b75..291b56448 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,7 +4,7 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -Release 2.3 (2019-08-??) +Release 2.3 (2019-09-04) This is primarily a bug fix release. However, it makes some incompatible changes: From 4caeefaf004c1a4fdd67485f0328a6741a9640fb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 14:06:52 +0200 Subject: [PATCH 0474/2548] Revert "Remove obsolete OpenSSL locking code" This reverts commit aeb695c0074b52772057b36f442a054f8d1a856d. --- src/libmain/shared.cc | 20 ++++++++++++++++++++ src/libutil/hash.cc | 4 ---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 910549583..0afddfb78 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,6 +15,8 @@ #include #include +#include + namespace nix { @@ -78,6 +80,20 @@ string getArg(const string & opt, } +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector opensslLocks; + +static void opensslLockCallback(int mode, int type, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} + + static void sigHandler(int signo) { } @@ -89,6 +105,10 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); + loadConfFile(); startSignalHandlerThread(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 362c537fe..1c14ebb18 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -13,10 +13,6 @@ #include #include -#if OPENSSL_VERSION_NUMBER < 0x10101000L -#error "Unsupported version of OpenSSL, you need at least 1.1.1" -#endif - namespace nix { From 6f88fed8195c43bc46e3f9e62273599f6cc205fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 14:14:03 +0200 Subject: [PATCH 0475/2548] Disable OpenSSL lock callback on OpenSSL >= 1.1.1 --- src/libmain/shared.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0afddfb78..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); From 252c78b288426bae2f379e8320d396b7eb40253b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:07 +0200 Subject: [PATCH 0476/2548] Tweak release notes --- doc/manual/release-notes/rl-2.3.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 291b56448..80fbd6396 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -13,9 +13,8 @@ incompatible changes: Nix now uses BSD file locks instead of POSIX file - locks. Since previous releases used POSIX file locks, you should - not use Nix 2.2 and previous releases at the same time on a Nix - store. + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store. @@ -47,9 +46,9 @@ incompatible changes: - nix: Add + The nix command has a new () flag to - print build log output to stderr rather than showing the last log + print build log output to stderr, rather than showing the last log line in the progress bar. To distinguish between concurrent builds, log lines are prefixed by the name of the package. @@ -57,7 +56,7 @@ incompatible changes: Builds are now executed in a pseudo-terminal, and the - TERM evnironment variable is set to + TERM environment variable is set to xterm-256color. This allows many programs (e.g. gcc, clang, cmake) to print colorized log output. From d20f814cdef55fd9d7cecf464b32fb8b5efecb1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 15:59:33 +0200 Subject: [PATCH 0477/2548] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c0943d3e9..7208c2182 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.3 \ No newline at end of file +2.4 \ No newline at end of file From 5dafde28dbec379678da6d033cdc5c48856babf5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 19:24:35 +0200 Subject: [PATCH 0478/2548] BinaryCacheStore: Add index-debug-info option This integrates the functionality of the index-debuginfo program in nixos-channel-scripts to maintain an index of DWARF debuginfo files in a format usable by dwarffs. Thus the debug info index is updated by Hydra rather than by the channel mirroring script. Example usage: $ nix copy --to 'file:///tmp/binary-cache?index-debug-info=true' /nix/store/vr9mhcch3fljzzkjld3kvkggvpq38cva-nix-2.2.2-debug $ cat /tmp/binary-cache/debuginfo/036b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug {"archive":"../nar/0313h2kdhk4v73xna9ysiksp2v8xrsk5xsw79mmwr3rg7byb4ka8.nar.xz","member":"lib/debug/.build-id/03/6b210b03bad75ab2d8fc80b7a146f98e7f1ecf.debug"} Fixes #3083. --- src/libstore/binary-cache-store.cc | 80 ++++++++++++++++++++---- src/libstore/binary-cache-store.hh | 1 + src/libstore/local-binary-cache-store.cc | 2 + 3 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 10cde8704..8e6f1f55d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,10 +10,13 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include - #include +#include + +#include namespace nix { @@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, narAccessor); - { auto res = jsonRoot.placeholder("root"); listNar(res, narAccessor, "", true); @@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refaddToCache(info.path, *nar, makeNarAccessor(nar)); - } - /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const refsize() / nar->size()) * 100.0) % duration); - /* Atomically write the NAR file. */ narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index af108880c..c77292294 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,6 +17,7 @@ public: const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting parallelCompression{this, false, "parallel-compression", diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b7001795b..9f99ee76d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -63,6 +63,8 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } From b9d6c6f000d2ba3efd7c124e46982425962f4fbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 21:44:40 +0200 Subject: [PATCH 0479/2548] upload-release.pl: Fix sshfs call --- maintainers/upload-release.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 1cdf5ed16..77534babb 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n"; File::Path::make_path($releasesDir); if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; + system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; } my $releaseDir = "$releasesDir/nix/$releaseName"; From a56b51a0ba7b0d6fdff7fd0127a118185b146f4f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2019 14:14:03 +0200 Subject: [PATCH 0480/2548] Disable OpenSSL lock callback on OpenSSL >= 1.1.1 --- src/libmain/shared.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0afddfb78..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); From 2fa7f2a56a5c2fe11c1a0daceee5cf0584b69be9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2019 17:15:09 +0200 Subject: [PATCH 0481/2548] Use git+ prefix in flake URI schemes Fixes #3045. --- src/libexpr/flake/flakeref.cc | 40 ++++++++++++++++++++------------- src/libexpr/flake/flakeref.hh | 20 ++++++++--------- src/libexpr/primops/fetchGit.cc | 2 ++ 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 253442566..4ce326c0b 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -27,7 +27,7 @@ const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; // URI stuff. -const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; +const static std::string schemeRegex = "[a-z+]+"; const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; @@ -120,21 +120,29 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) data = d; } - else if (std::regex_match(uri.c_str(), match, uriRegex) - && (match[2] == "file" || hasSuffix(match[4], ".git"))) - { - IsGit d; - d.uri = match[1]; - for (auto & param : params) { - if (handleGitParams(param.first, param.second)) - ; - else - // FIXME: should probably pass through unknown parameters - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - if (rev && !ref) - throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); - data = d; + else if (std::regex_match(uri.c_str(), match, uriRegex)) { + auto & scheme = match[2]; + if (scheme == "git" || + scheme == "git+http" || + scheme == "git+https" || + scheme == "git+ssh" || + scheme == "git+file" || + scheme == "file") + { + IsGit d; + d.uri = match[1]; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + // FIXME: should probably pass through unknown parameters + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + if (rev && !ref) + throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); + data = d; + } else + throw BadFlakeRef("unsupported URI scheme '%s' in flake reference '%s'", scheme, uri); } else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 9ddc227bb..6b47330a7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -52,28 +52,28 @@ namespace nix { github:edolstra/dwarffs/unstable github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 - * https:///.git(\?attr(&attr)*)? - ssh:///.git(\?attr(&attr)*)? - git:///.git(\?attr(&attr)*)? + * git+https:///(\?attr(&attr)*)? + git+ssh:///(\?attr(&attr)*)? + git:///(\?attr(&attr)*)? file:///(\?attr(&attr)*)? where 'attr' is one of: rev= ref= - A Git repository fetched through https. Note that the path must - end in ".git". The default for "ref" is "master". + A Git repository fetched through https. The default for "ref" is + "master". Examples: - https://example.org/my/repo.git - https://example.org/my/repo.git?ref=release-1.2.3 - https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + git+https://example.org/my/repo.git + git+https://example.org/my/repo.git?ref=release-1.2.3 + git+https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 - * /path.git(\?attr(&attr)*)? + * /path(\?attr(&attr)*)? - Like file://path.git, but if no "ref" or "rev" is specified, the + Like file://path, but if no "ref" or "rev" is specified, the (possibly dirty) working tree will be used. Using a working tree is not allowed in pure evaluation mode. diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 931eac95f..48d84c4a1 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -25,6 +25,8 @@ GitInfo exportGit(ref store, std::string uri, { assert(!rev || rev->type == htSHA1); + if (hasPrefix(uri, "git+")) uri = std::string(uri, 4); + bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git"); // If this is a local directory (but not a file:// URI) and no ref From c87840ae14eea84b5910cb0b188ec3fb32cc1466 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Sep 2019 16:34:44 +0200 Subject: [PATCH 0482/2548] Don't allow arbitrary computations in flake attributes E.g. you can write 'edition = 201909' but not 'edition = 201909 + 0'. Fixes #3075. --- src/libexpr/eval.cc | 23 ++++++++++++++++++----- src/libexpr/eval.hh | 1 + src/libexpr/flake/flake.cc | 36 +++++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fa79b0d5e..25b50da7e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -141,12 +141,12 @@ const Value *getPrimOp(const Value &v) { } -string showType(const Value & v) +string showType(ValueType type) { - switch (v.type) { + switch (type) { case tInt: return "an integer"; case tBool: return "a boolean"; - case tString: return v.string.context ? "a string with context" : "a string"; + case tString: return "a string"; case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; @@ -155,14 +155,27 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; + case tPrimOp: return "a built-in function"; + case tPrimOpApp: return "a partially applied built-in function"; + case tExternal: return "an external value"; + case tFloat: return "a float"; + } + abort(); +} + + +string showType(const Value & v) +{ + switch (v.type) { + case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); - case tFloat: return "a float"; + default: + return showType(v.type); } - abort(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 5e976f196..468a826ca 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -338,6 +338,7 @@ private: /* Return a string representing the type of the value `v'. */ +string showType(ValueType type); string showType(const Value & v); /* Decode a context string ‘!!’ into a pair (value.thunk.expr)) || + (type == tLambda && dynamic_cast(value.thunk.expr)))) + state.forceValue(value, pos); + if (value.type != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type), pos); +} + Flake getFlake(EvalState & state, const FlakeRef & flakeRef) { SourceInfo sourceInfo = fetchFlake(state, flakeRef); @@ -219,9 +231,10 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); Value vInfo; + // FIXME: don't evaluate vInfo. state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack - state.forceAttrs(vInfo); + expectType(state, tAttrs, vInfo, Pos(state.symbols.create(realFlakeFile), 0, 0)); auto sEdition = state.symbols.create("edition"); auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon @@ -231,7 +244,8 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) edition = vInfo.attrs->get(sEpoch); if (edition) { - flake.edition = state.forceInt(*(**edition).value, *(**edition).pos); + expectType(state, tInt, *(**edition).value, *(**edition).pos); + flake.edition = (**edition).value->integer; if (flake.edition > 201909) throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", flakeRef, flake.edition); if (flake.edition < 201909) @@ -239,26 +253,30 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) } else throw Error("flake '%s' lacks attribute 'edition'", flakeRef); - if (auto description = vInfo.attrs->get(state.sDescription)) - flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, tString, *(**description).value, *(**description).pos); + flake.description = (**description).value->string.s; + } auto sInputs = state.symbols.create("inputs"); auto sUri = state.symbols.create("uri"); auto sFlake = state.symbols.create("flake"); if (std::optional inputs = vInfo.attrs->get(sInputs)) { - state.forceAttrs(*(**inputs).value, *(**inputs).pos); + expectType(state, tAttrs, *(**inputs).value, *(**inputs).pos); for (Attr inputAttr : *(*(**inputs).value).attrs) { - state.forceAttrs(*inputAttr.value, *inputAttr.pos); + expectType(state, tAttrs, *inputAttr.value, *inputAttr.pos); FlakeInput input(FlakeRef(inputAttr.name)); for (Attr attr : *(inputAttr.value->attrs)) { if (attr.name == sUri) { - input.ref = state.forceStringNoCtx(*attr.value, *attr.pos); + expectType(state, tString, *attr.value, *attr.pos); + input.ref = std::string(attr.value->string.s); } else if (attr.name == sFlake) { - input.isFlake = state.forceBool(*attr.value, *attr.pos); + expectType(state, tBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; } else throw Error("flake input '%s' has an unsupported attribute '%s', at %s", inputAttr.name, attr.name, *attr.pos); @@ -271,7 +289,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { - state.forceFunction(*(**outputs).value, *(**outputs).pos); + expectType(state, tLambda, *(**outputs).value, *(**outputs).pos); flake.vOutputs = (**outputs).value; if (flake.vOutputs->lambda.fun->matchAttrs) { From f97d3753a13f0ff916d83dbea4fe7dae7194f903 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Sep 2019 17:34:38 +0200 Subject: [PATCH 0483/2548] Require flake.nix to be an attrset (not a non-trivial thunk) --- src/libexpr/eval.cc | 19 ++++++++++++++++++- src/libexpr/eval.hh | 5 +++-- src/libexpr/flake/flake.cc | 7 ++----- src/libexpr/value.hh | 5 +++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 25b50da7e..ec751ad31 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -179,6 +179,18 @@ string showType(const Value & v) } +bool Value::isTrivial() const +{ + return + type != tApp + && type != tPrimOpApp + && (type != tThunk + || (dynamic_cast(thunk.expr) + && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast(thunk.expr)); +} + + #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ static void * oomHandler(size_t requested) @@ -749,7 +761,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v) +void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -778,6 +790,11 @@ void EvalState::evalFile(const Path & path_, Value & v) fileParseCache[path2] = e; try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast(e))) + throw Error("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 468a826ca..16350a5bf 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -157,8 +157,9 @@ public: Expr * parseStdin(); /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + form. Optionally enforce that the top-level expression is + trivial (i.e. doesn't require arbitrary computation). */ + void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); void resetFileCache(); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c10906731..050e65259 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -198,9 +198,7 @@ static SourceInfo fetchFlake(EvalState & state, const FlakeRef & resolvedRef) static void expectType(EvalState & state, ValueType type, Value & value, const Pos & pos) { - if (value.type == tThunk && - ((type == tAttrs && dynamic_cast(value.thunk.expr)) || - (type == tLambda && dynamic_cast(value.thunk.expr)))) + if (value.type == tThunk && value.isTrivial()) state.forceValue(value, pos); if (value.type != type) throw Error("expected %s but got %s at %s", @@ -231,8 +229,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); Value vInfo; - // FIXME: don't evaluate vInfo. - state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack + state.evalFile(realFlakeFile, vInfo, true); // FIXME: symlink attack expectType(state, tAttrs, vInfo, Pos(state.symbols.create(realFlakeFile), 0, 0)); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index e1ec87d3b..bdf2cdde1 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -170,6 +170,11 @@ struct Value { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } + + /* Check whether forcing this value requires a trivial amount of + computation. In particular, function applications are + non-trivial. */ + bool isTrivial() const; }; From dc3f52a1447df8523f44c89e25e48e8b7f5341a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 14:52:22 +0200 Subject: [PATCH 0484/2548] nix flake check: Check overlays --- src/nix/flake.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 4129ef323..10ce9addc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -280,6 +280,22 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON } }; + auto checkOverlay = [&](const std::string & attrPath, Value & v) { + try { + state->forceValue(v); + if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + throw Error("overlay does not take an argument named 'final'"); + auto body = dynamic_cast(v.lambda.fun->body); + if (!body || body->matchAttrs || std::string(body->arg) != "prev") + throw Error("overlay does not take an argument named 'prev'"); + // FIXME: if we have a 'nixpkgs' input, use it to + // evaluate the overlay. + } catch (Error & e) { + e.addPrefix(fmt("while checking the overlay '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -326,6 +342,9 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON // FIXME: do getDerivations? ; + else if (name == "overlay") + checkOverlay(name, vProvide); + else warn("unknown flake output '%s'", name); From 4b9dee6bcca48bd60f341cb07273a33e632bafc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 15:25:10 +0200 Subject: [PATCH 0485/2548] nix flake check: Do some basic checks on NixOS modules Also show more position info. --- src/nix/flake.cc | 87 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 23 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 10ce9addc..b29aa212c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -170,7 +170,7 @@ struct CmdFlakeUpdate : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { state.forceAttrs(vFlake); @@ -179,7 +179,7 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, state.forceAttrs(*vOutputs); for (auto & attr : *vOutputs->attrs) - callback(attr.name, *attr.value); + callback(attr.name, *attr.value, *attr.pos); } struct CmdFlakeInfo : FlakeCommand, MixJSON @@ -207,7 +207,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vProvide) { + [&](const std::string & name, Value & vProvide, const Pos & pos) { auto provide = nlohmann::json::object(); if (name == "checks" || name == "packages") { @@ -251,7 +251,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = resolveFlake(); - auto checkDerivation = [&](const std::string & attrPath, Value & v) { + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) @@ -259,14 +259,14 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON // FIXME: check meta attributes return drvInfo->queryDrvPath(); } catch (Error & e) { - e.addPrefix(fmt("while checking the derivation '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + e.addPrefix(fmt("while checking the derivation '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); throw; } }; PathSet drvPaths; - auto checkApp = [&](const std::string & attrPath, Value & v) { + auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { auto app = App(*state, v); for (auto & i : app.context) { @@ -275,12 +275,12 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON drvPaths.insert(drvPath + "!" + outputName); } } catch (Error & e) { - e.addPrefix(fmt("while checking the app definition '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + e.addPrefix(fmt("while checking the app definition '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); throw; } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v) { + auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v); if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") @@ -291,7 +291,31 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { - e.addPrefix(fmt("while checking the overlay '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", attrPath)); + e.addPrefix(fmt("while checking the overlay '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v); + if (v.type == tLambda) { + if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + throw Error("module must match an open attribute set ('{ config, ... }')"); + } else if (v.type == tAttrs) { + for (auto & attr : *v.attrs) + try { + state->forceValue(*attr.value); + } catch (Error & e) { + e.addPrefix(fmt("while evaluating the option '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attr.name, *attr.pos)); + throw; + } + } else + throw Error("module must be a function or an attribute set"); + // FIXME: if we have a 'nixpkgs' input, use it to + // check the module. + } catch (Error & e) { + e.addPrefix(fmt("while checking the NixOS module '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); throw; } }; @@ -304,46 +328,63 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vProvide) { + [&](const std::string & name, Value & vOutput, const Pos & pos) { Activity act(*logger, lvlChatty, actUnknown, fmt("checking flake output '%s'", name)); try { - state->forceValue(vProvide); + state->forceValue(vOutput); if (name == "checks") { - state->forceAttrs(vProvide); - for (auto & aCheck : *vProvide.attrs) + state->forceAttrs(vOutput); + for (auto & attr : *vOutput.attrs) drvPaths.insert(checkDerivation( - name + "." + (std::string) aCheck.name, *aCheck.value)); + name + "." + (std::string) attr.name, *attr.value, *attr.pos)); } else if (name == "packages") { - state->forceAttrs(vProvide); - for (auto & aCheck : *vProvide.attrs) + state->forceAttrs(vOutput); + for (auto & attr : *vOutput.attrs) checkDerivation( - name + "." + (std::string) aCheck.name, *aCheck.value); + name + "." + (std::string) attr.name, *attr.value, *attr.pos); } else if (name == "apps") { - state->forceAttrs(vProvide); - for (auto & aCheck : *vProvide.attrs) + state->forceAttrs(vOutput); + for (auto & attr : *vOutput.attrs) checkApp( - name + "." + (std::string) aCheck.name, *aCheck.value); + name + "." + (std::string) attr.name, *attr.value, *attr.pos); } else if (name == "defaultPackage" || name == "devShell") - checkDerivation(name, vProvide); + checkDerivation(name, vOutput, pos); else if (name == "defaultApp") - checkApp(name, vProvide); + checkApp(name, vOutput, pos); else if (name == "legacyPackages") // FIXME: do getDerivations? ; else if (name == "overlay") - checkOverlay(name, vProvide); + checkOverlay(name, vOutput, pos); + + else if (name == "overlays") { + state->forceAttrs(vOutput); + for (auto & attr : *vOutput.attrs) + checkOverlay(name + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + else if (name == "nixosModule") + checkModule(name, vOutput, pos); + + else if (name == "nixosModules") { + state->forceAttrs(vOutput); + for (auto & attr : *vOutput.attrs) + checkModule(name + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } else warn("unknown flake output '%s'", name); From f3f854dac15d5e5318cd0398fd50a8d8db4f6fd0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 16:03:03 +0200 Subject: [PATCH 0486/2548] nix flake check: Add some tests --- tests/flakes.sh | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/flakes.sh b/tests/flakes.sh index 8594a4c55..7f41d5d7e 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -344,3 +344,88 @@ nix flake --flake-registry $registry check $flake3Dir rm -rf $TEST_ROOT/flake1-v2 nix flake clone --flake-registry $registry flake1 $TEST_ROOT/flake1-v2 + +# More 'nix flake check' tests. +cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Tue, 10 Sep 2019 16:06:43 +0200 Subject: [PATCH 0487/2548] Test quoted attrpaths Issue #3076. --- tests/flakes.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/flakes.sh b/tests/flakes.sh index 7f41d5d7e..f3baf8cef 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -183,7 +183,7 @@ cat > $flake3Dir/flake.nix < Date: Tue, 10 Sep 2019 17:39:34 +0200 Subject: [PATCH 0488/2548] flake.nix: Remove VM tests from 'checks' --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 06a5515c5..f0d928494 100644 --- a/flake.nix +++ b/flake.nix @@ -13,8 +13,6 @@ checks = { binaryTarball = hydraJobs.binaryTarball.x86_64-linux; perlBindings = hydraJobs.perlBindings.x86_64-linux; - inherit (hydraJobs.tests) remoteBuilds nix-copy-closure; - setuid = hydraJobs.tests.setuid.x86_64-linux; }; packages = { From 55e55b34e6ea770ef1310dffb96f476bf37d460c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2019 17:39:55 +0200 Subject: [PATCH 0489/2548] nix flake check: Check hydraJobs --- src/nix/flake.cc | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b29aa212c..5fd3f5508 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -211,7 +211,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON auto provide = nlohmann::json::object(); if (name == "checks" || name == "packages") { - state->forceAttrs(vProvide); + state->forceAttrs(vProvide, pos); for (auto & aCheck : *vProvide.attrs) provide[aCheck.name] = nlohmann::json::object(); } @@ -282,7 +282,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { - state->forceValue(v); + state->forceValue(v, pos); if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast(v.lambda.fun->body); @@ -298,14 +298,14 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { - state->forceValue(v); + state->forceValue(v, pos); if (v.type == tLambda) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); } else if (v.type == tAttrs) { for (auto & attr : *v.attrs) try { - state->forceValue(*attr.value); + state->forceValue(*attr.value, *attr.pos); } catch (Error & e) { e.addPrefix(fmt("while evaluating the option '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attr.name, *attr.pos)); throw; @@ -320,6 +320,28 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON } }; + std::function checkHydraJobs; + + checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceAttrs(v, pos); + + if (state->isDerivation(v)) + throw Error("jobset should not be a derivation at top-level"); + + for (auto & attr : *v.attrs) { + state->forceAttrs(*attr.value, *attr.pos); + if (!state->isDerivation(*attr.value)) + checkHydraJobs(attrPath + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + } catch (Error & e) { + e.addPrefix(fmt("while checking the Hydra jobset '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -333,24 +355,24 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON fmt("checking flake output '%s'", name)); try { - state->forceValue(vOutput); + state->forceValue(vOutput, pos); if (name == "checks") { - state->forceAttrs(vOutput); + state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) drvPaths.insert(checkDerivation( name + "." + (std::string) attr.name, *attr.value, *attr.pos)); } else if (name == "packages") { - state->forceAttrs(vOutput); + state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) checkDerivation( name + "." + (std::string) attr.name, *attr.value, *attr.pos); } else if (name == "apps") { - state->forceAttrs(vOutput); + state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) checkApp( name + "." + (std::string) attr.name, *attr.value, *attr.pos); @@ -370,7 +392,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON checkOverlay(name, vOutput, pos); else if (name == "overlays") { - state->forceAttrs(vOutput); + state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) checkOverlay(name + "." + (std::string) attr.name, *attr.value, *attr.pos); @@ -380,12 +402,15 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON checkModule(name, vOutput, pos); else if (name == "nixosModules") { - state->forceAttrs(vOutput); + state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) checkModule(name + "." + (std::string) attr.name, *attr.value, *attr.pos); } + else if (name == "hydraJobs") + checkHydraJobs(name, vOutput, pos); + else warn("unknown flake output '%s'", name); From 92ede15dd902f7c1d2771c194b8bb73fe406840f Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:11:37 +0200 Subject: [PATCH 0490/2548] docs: Fix a typo in github in an example --- doc/manual/expressions/builtins.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 4c1d618e9..39407c9ac 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -448,7 +448,7 @@ stdenv.mkDerivation { … } Fetching an arbitrary ref builtins.fetchGit { - url = "https://gitub.com/NixOS/nix.git"; + url = "https://github.com/NixOS/nix.git"; ref = "refs/heads/0.5-release"; } From ae244af242ca3621e5a3b9196f27d9fcbf297266 Mon Sep 17 00:00:00 2001 From: Julien Tanguy Date: Wed, 11 Sep 2019 14:18:47 +0200 Subject: [PATCH 0491/2548] docs: Use the explicit ref for fetchGit with a tag With the merge of #2582, the syntax "tags/1.9" for refs does not work anymore. However, the new syntax "refs/tags/1.9" seems to support annotated tags, such as "refs/tags/2.0". Closes #2385. --- doc/manual/expressions/builtins.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 39407c9ac..191fa7810 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -499,11 +499,8 @@ stdenv.mkDerivation { … } Fetching a tag builtins.fetchGit { url = "https://github.com/nixos/nix.git"; - ref = "tags/1.9"; + ref = "refs/tags/1.9"; } - Due to a bug (#2385), - only non-annotated tags can be fetched. From 6b83174ffffbdfc3f876d94d5178e0b83f675cae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 13 Sep 2019 20:05:27 +0200 Subject: [PATCH 0492/2548] std::uncaught_exception() -> std::uncaught_exceptions() The former is deprecated in C++17. Fixes a clang warning. --- src/libstore/remote-store.cc | 2 +- src/libutil/json.hh | 2 +- src/libutil/util.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e38fe49a7..89b699414 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -228,7 +228,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exception()) { + if (!daemonException && std::uncaught_exceptions()) { handle.markBad(); debug("closing daemon connection because of an exception"); } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..45a22f011 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -170,7 +170,7 @@ public: ~JSONPlaceholder() { - assert(!first || std::uncaught_exception()); + assert(!first || std::uncaught_exceptions()); } template diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1b7449991..e65f8ee56 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1169,7 +1169,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exception()) { + if (!interruptThrown && !std::uncaught_exceptions()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } From 9533d85ce009c65722a2addc55b64ee51f15dea3 Mon Sep 17 00:00:00 2001 From: Alexandre Esteves Date: Tue, 17 Sep 2019 23:40:49 +0100 Subject: [PATCH 0493/2548] Move 'builtins.splitVersion' to position respecting alphabetical order --- doc/manual/expressions/builtins.xml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 191fa7810..9bd86e120 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -170,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" - - builtins.splitVersion - s - - Split a string representing a version into its - components, by the same version splitting logic underlying the - version comparison in - nix-env -u. - - - - builtins.concatLists lists @@ -1272,6 +1260,19 @@ Evaluates to [ " " [ "FOO" ] " " ]. + + + builtins.splitVersion + s + + Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in + nix-env -u. + + + + builtins.stringLength e From c67407172d8383394f4962ad177c84bf04529e5e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Sep 2019 21:17:27 +0200 Subject: [PATCH 0494/2548] Record original flakerefs in the lock file again If 'input..uri' changes, then the entry in the lockfile for input should be considered stale. Also print some messages when lock file entries are added/updated. --- flake.lock | 7 +++--- src/libexpr/flake/flake.cc | 43 ++++++++++++++++++++++++----------- src/libexpr/flake/flake.hh | 2 +- src/libexpr/flake/lockfile.cc | 6 +++-- src/libexpr/flake/lockfile.hh | 6 ++--- src/nix/flake.cc | 7 +++--- 6 files changed, 45 insertions(+), 26 deletions(-) diff --git a/flake.lock b/flake.lock index fa5649c03..70a433b26 100644 --- a/flake.lock +++ b/flake.lock @@ -2,9 +2,10 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", - "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" + "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", + "originalUri": "nixpkgs", + "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" } }, - "version": 2 + "version": 3 } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 050e65259..accdb4194 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -205,8 +205,10 @@ static void expectType(EvalState & state, ValueType type, showType(type), showType(value.type), pos); } -Flake getFlake(EvalState & state, const FlakeRef & flakeRef) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { + auto flakeRef = maybeLookupFlake(state, originalRef, allowLookup); + SourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); @@ -223,7 +225,7 @@ Flake getFlake(EvalState & state, const FlakeRef & flakeRef) if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); - Flake flake(flakeRef, sourceInfo); + Flake flake(originalRef, sourceInfo); if (!pathExists(realFlakeFile)) throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); @@ -358,6 +360,7 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) Note that this is lazy: we only recursively fetch inputs that are not in the lockfile yet. */ static std::pair updateLocks( + const std::string & inputPath, EvalState & state, const Flake & flake, HandleLockFile handleLockFile, @@ -366,23 +369,36 @@ static std::pair updateLocks( { LockedInput newEntry( flake.sourceInfo.resolvedRef, + flake.originalRef, flake.sourceInfo.narHash); for (auto & [id, input] : flake.inputs) { + auto inputPath2 = (inputPath.empty() ? "" : inputPath + "/") + id; auto i = oldEntry.inputs.find(id); - if (i != oldEntry.inputs.end()) { + if (i != oldEntry.inputs.end() && i->second.originalRef == input.ref) { newEntry.inputs.insert_or_assign(id, i->second); } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake input '%s' in pure mode", id); - if (input.isFlake) + if (input.isFlake) { + auto actualInput = getFlake(state, input.ref, + allowedToUseRegistries(handleLockFile, false)); + if (i == oldEntry.inputs.end()) + printMsg(lvlWarn, "mapped flake input '%s' to '%s'", + inputPath2, actualInput.sourceInfo.resolvedRef); + else + printMsg(lvlWarn, "updated flake input '%s' from '%s' to '%s'", + inputPath2, i->second.originalRef, actualInput.sourceInfo.resolvedRef); newEntry.inputs.insert_or_assign(id, - updateLocks(state, - getFlake(state, maybeLookupFlake(state, input.ref, allowedToUseRegistries(handleLockFile, false))), - handleLockFile, {}, false).second); - else { - auto sourceInfo = getNonFlake(state, maybeLookupFlake(state, input.ref, allowedToUseRegistries(handleLockFile, false))); - newEntry.inputs.insert_or_assign(id, LockedInput(sourceInfo.resolvedRef, sourceInfo.narHash)); + updateLocks(inputPath2, state, actualInput, handleLockFile, {}, false).second); + } else { + auto sourceInfo = getNonFlake(state, + maybeLookupFlake(state, input.ref, + allowedToUseRegistries(handleLockFile, false))); + printMsg(lvlWarn, "mapped flake input '%s' to '%s'", + inputPath2, sourceInfo.resolvedRef); + newEntry.inputs.insert_or_assign(id, + LockedInput(sourceInfo.resolvedRef, input.ref, sourceInfo.narHash)); } } } @@ -394,7 +410,8 @@ static std::pair updateLocks( and optionally write it to file, it the flake is writable. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { - auto flake = getFlake(state, maybeLookupFlake(state, topRef, allowedToUseRegistries(handleLockFile, true))); + auto flake = getFlake(state, topRef, + allowedToUseRegistries(handleLockFile, true)); LockFile oldLockFile; @@ -407,7 +424,7 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc } LockFile lockFile(updateLocks( - state, flake, handleLockFile, oldLockFile, true).second); + "", state, flake, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { @@ -476,7 +493,7 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V assert(lazyInput->lockedInput.ref.isImmutable()); if (lazyInput->isFlake) { - auto flake = getFlake(state, lazyInput->lockedInput.ref); + auto flake = getFlake(state, lazyInput->lockedInput.ref, false); if (flake.sourceInfo.narHash != lazyInput->lockedInput.narHash) throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 8f05e9799..b366e650b 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -78,7 +78,7 @@ struct Flake : originalRef(origRef), sourceInfo(sourceInfo) {}; }; -Flake getFlake(EvalState &, const FlakeRef &); +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); /* If 'allowLookup' is true, then resolve 'flakeRef' using the registries. */ diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index f32d752f9..039b7a7c1 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -6,6 +6,7 @@ namespace nix::flake { LockedInput::LockedInput(const nlohmann::json & json) : LockedInputs(json) , ref(json["uri"]) + , originalRef(json["originalUri"]) , narHash(Hash((std::string) json["narHash"])) { if (!ref.isImmutable()) @@ -16,6 +17,7 @@ nlohmann::json LockedInput::toJson() const { auto json = LockedInputs::toJson(); json["uri"] = ref.to_string(); + json["originalUri"] = originalRef.to_string(); json["narHash"] = narHash.to_string(SRI); return json; } @@ -54,7 +56,7 @@ bool LockedInputs::isDirty() const nlohmann::json LockFile::toJson() const { auto json = LockedInputs::toJson(); - json["version"] = 2; + json["version"] = 3; return json; } @@ -64,7 +66,7 @@ LockFile LockFile::read(const Path & path) auto json = nlohmann::json::parse(readFile(path)); auto version = json.value("version", 0); - if (version != 2) + if (version != 3) throw Error("lock file '%s' has unsupported version %d", path, version); return LockFile(json); diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 084eabc1a..ab81eac8b 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -30,11 +30,11 @@ struct LockedInputs /* Lock file information about a flake input. */ struct LockedInput : LockedInputs { - FlakeRef ref; + FlakeRef ref, originalRef; Hash narHash; - LockedInput(const FlakeRef & ref, const Hash & narHash) - : ref(ref), narHash(narHash) + LockedInput(const FlakeRef & ref, const FlakeRef & originalRef, const Hash & narHash) + : ref(ref), originalRef(originalRef), narHash(narHash) { assert(ref.isImmutable()); }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5fd3f5508..599bf12eb 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -38,8 +38,7 @@ public: Flake getFlake() { auto evalState = getEvalState(); - return flake::getFlake(*evalState, - maybeLookupFlake(*evalState, getFlakeRef(), useRegistries)); + return flake::getFlake(*evalState, getFlakeRef(), useRegistries); } ResolvedFlake resolveFlake() @@ -500,13 +499,13 @@ struct CmdFlakePin : virtual Args, EvalCommand FlakeRegistry userRegistry = *readRegistry(userRegistryPath); auto it = userRegistry.entries.find(FlakeRef(alias)); if (it != userRegistry.entries.end()) { - it->second = getFlake(*evalState, maybeLookupFlake(*evalState, it->second, true)).sourceInfo.resolvedRef; + it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; writeRegistry(userRegistry, userRegistryPath); } else { std::shared_ptr globalReg = evalState->getGlobalFlakeRegistry(); it = globalReg->entries.find(FlakeRef(alias)); if (it != globalReg->entries.end()) { - auto newRef = getFlake(*evalState, maybeLookupFlake(*evalState, it->second, true)).sourceInfo.resolvedRef; + auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef; userRegistry.entries.insert_or_assign(alias, newRef); writeRegistry(userRegistry, userRegistryPath); } else From 619cc4af855fab7b0400586a4fd40745b23e72ad Mon Sep 17 00:00:00 2001 From: zimbatm Date: Wed, 18 Sep 2019 11:40:11 +0000 Subject: [PATCH 0495/2548] function-trace: always show the trace If the user invokes nix with --trace-function-calls it means that they want to see the trace. --- contrib/stack-collapse.py | 5 ++--- src/libexpr/function-trace.hh | 4 ++-- tests/function-trace.sh | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py index ee77e1589..f5602c95c 100755 --- a/contrib/stack-collapse.py +++ b/contrib/stack-collapse.py @@ -1,12 +1,11 @@ #!/usr/bin/env nix-shell #!nix-shell -i python3 -p python3 --pure -# To be used with `--trace-function-calls` and `-vvvv` and -# `flamegraph.pl`. +# To be used with `--trace-function-calls` and `flamegraph.pl`. # # For example: # -# nix-instantiate --trace-function-calls -vvvv '' -A hello 2> nix-function-calls.trace +# nix-instantiate --trace-function-calls '' -A hello 2> nix-function-calls.trace # ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded # nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index 8234b7603..8b0ec848d 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -12,13 +12,13 @@ struct FunctionCallTrace FunctionCallTrace(const Pos & pos) : pos(pos) { auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto ns = std::chrono::duration_cast(duration); - vomit("function-trace entered %1% at %2%", pos, ns.count()); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); } ~FunctionCallTrace() { auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); auto ns = std::chrono::duration_cast(duration); - vomit("function-trace exited %1% at %2%", pos, ns.count()); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); } }; } diff --git a/tests/function-trace.sh b/tests/function-trace.sh index f7d93b435..182a4d5c2 100755 --- a/tests/function-trace.sh +++ b/tests/function-trace.sh @@ -8,7 +8,6 @@ expect_trace() { actual=$( nix-instantiate \ --trace-function-calls \ - -vvvv \ --expr "$expr" 2>&1 \ | grep "function-trace" \ | sed -e 's/ [0-9]*$//' From aeb7148afd56b228604b79373a45793d36d660a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Sep 2019 23:59:45 +0200 Subject: [PATCH 0496/2548] Some effort to minimize flake dependencies For example, if the top-level flake depends on "nixpkgs/release-19.03", and one of its dependencies depends on "nixpkgs", then the latter will be mapped to "nixpkgs/release-19.03", rather than whatever the default branch of "nixpkgs" is. Thus you get only one "nixpkgs" dependency rather than two. This currently only works in a breadth-first way, so the other way around (i.e. if the top-level flake depends on "nixpkgs", and a dependency depends on "nixpkgs/release-19.03") still results in two "nixpkgs" dependencies. --- src/libexpr/flake/flake.cc | 90 +++++++++++++++++++++++++++-------- src/libexpr/flake/flake.hh | 7 --- src/libexpr/flake/flakeref.cc | 17 +++++++ src/libexpr/flake/flakeref.hh | 6 +++ 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index accdb4194..d9bbf85c2 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -112,7 +112,9 @@ static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const return flakeRef; } -FlakeRef maybeLookupFlake( +/* If 'allowLookup' is true, then resolve 'flakeRef' using the + registries. */ +static FlakeRef maybeLookupFlake( EvalState & state, const FlakeRef & flakeRef, bool allowLookup) @@ -126,6 +128,23 @@ FlakeRef maybeLookupFlake( return flakeRef; } +typedef std::vector> RefMap; + +static FlakeRef lookupInRefMap( + const RefMap & refMap, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : refMap) { + if (flakeRef.contains(i.first)) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second); + return i.second; + } + } + + return flakeRef; +} static SourceInfo fetchFlake(EvalState & state, const FlakeRef & resolvedRef) { @@ -205,15 +224,21 @@ static void expectType(EvalState & state, ValueType type, showType(type), showType(value.type), pos); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +static Flake getFlake(EvalState & state, const FlakeRef & originalRef, + bool allowLookup, RefMap & refMap) { - auto flakeRef = maybeLookupFlake(state, originalRef, allowLookup); + auto flakeRef = lookupInRefMap(refMap, + maybeLookupFlake(state, + lookupInRefMap(refMap, originalRef), allowLookup)); SourceInfo sourceInfo = fetchFlake(state, flakeRef); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; + refMap.push_back({originalRef, resolvedRef}); + refMap.push_back({flakeRef, resolvedRef}); + state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) @@ -314,13 +339,27 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return flake; } -static SourceInfo getNonFlake(EvalState & state, const FlakeRef & flakeRef) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { + RefMap refMap; + return getFlake(state, originalRef, allowLookup, refMap); +} + +static SourceInfo getNonFlake(EvalState & state, const FlakeRef & originalRef, + bool allowLookup, RefMap & refMap) +{ + auto flakeRef = lookupInRefMap(refMap, + maybeLookupFlake(state, + lookupInRefMap(refMap, originalRef), allowLookup)); + auto sourceInfo = fetchFlake(state, flakeRef); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; + refMap.push_back({originalRef, resolvedRef}); + refMap.push_back({flakeRef, resolvedRef}); + state.store->assertStorePath(sourceInfo.storePath); if (state.allowedPaths) @@ -360,6 +399,7 @@ bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) Note that this is lazy: we only recursively fetch inputs that are not in the lockfile yet. */ static std::pair updateLocks( + RefMap & refMap, const std::string & inputPath, EvalState & state, const Flake & flake, @@ -372,6 +412,8 @@ static std::pair updateLocks( flake.originalRef, flake.sourceInfo.narHash); + std::vector> postponed; + for (auto & [id, input] : flake.inputs) { auto inputPath2 = (inputPath.empty() ? "" : inputPath + "/") + id; auto i = oldEntry.inputs.find(id); @@ -380,29 +422,36 @@ static std::pair updateLocks( } else { if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) throw Error("cannot update flake input '%s' in pure mode", id); - if (input.isFlake) { - auto actualInput = getFlake(state, input.ref, - allowedToUseRegistries(handleLockFile, false)); + + auto warn = [&](const SourceInfo & sourceInfo) { if (i == oldEntry.inputs.end()) - printMsg(lvlWarn, "mapped flake input '%s' to '%s'", - inputPath2, actualInput.sourceInfo.resolvedRef); + printInfo("mapped flake input '%s' to '%s'", + inputPath2, sourceInfo.resolvedRef); else printMsg(lvlWarn, "updated flake input '%s' from '%s' to '%s'", - inputPath2, i->second.originalRef, actualInput.sourceInfo.resolvedRef); - newEntry.inputs.insert_or_assign(id, - updateLocks(inputPath2, state, actualInput, handleLockFile, {}, false).second); + inputPath2, i->second.originalRef, sourceInfo.resolvedRef); + }; + + if (input.isFlake) { + auto actualInput = getFlake(state, input.ref, + allowedToUseRegistries(handleLockFile, false), refMap); + warn(actualInput.sourceInfo); + postponed.push_back([&, id{id}, inputPath2, actualInput]() { + newEntry.inputs.insert_or_assign(id, + updateLocks(refMap, inputPath2, state, actualInput, handleLockFile, {}, false).second); + }); } else { - auto sourceInfo = getNonFlake(state, - maybeLookupFlake(state, input.ref, - allowedToUseRegistries(handleLockFile, false))); - printMsg(lvlWarn, "mapped flake input '%s' to '%s'", - inputPath2, sourceInfo.resolvedRef); + auto sourceInfo = getNonFlake(state, input.ref, + allowedToUseRegistries(handleLockFile, false), refMap); + warn(sourceInfo); newEntry.inputs.insert_or_assign(id, LockedInput(sourceInfo.resolvedRef, input.ref, sourceInfo.narHash)); } } } + for (auto & f : postponed) f(); + return {flake, newEntry}; } @@ -423,8 +472,10 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } + RefMap refMap; + LockFile lockFile(updateLocks( - "", state, flake, handleLockFile, oldLockFile, true).second); + refMap, "", state, flake, handleLockFile, oldLockFile, true).second); if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { @@ -500,7 +551,8 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V callFlake(state, flake, lazyInput->lockedInput, v); } else { - auto sourceInfo = getNonFlake(state, lazyInput->lockedInput.ref); + RefMap refMap; + auto sourceInfo = getNonFlake(state, lazyInput->lockedInput.ref, false, refMap); if (sourceInfo.narHash != lazyInput->lockedInput.narHash) throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", sourceInfo.resolvedRef); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b366e650b..63d848889 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -80,13 +80,6 @@ struct Flake Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* If 'allowLookup' is true, then resolve 'flakeRef' using the - registries. */ -FlakeRef maybeLookupFlake( - EvalState & state, - const FlakeRef & flakeRef, - bool allowLookup); - /* Fingerprint of a locked flake; used as a cache key. */ typedef Hash Fingerprint; diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 4ce326c0b..4930d03ce 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -255,6 +255,23 @@ FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. return result; } +bool FlakeRef::contains(const FlakeRef & other) const +{ + if (!(data == other.data)) + return false; + + if (ref && ref != other.ref) + return false; + + if (rev && rev != other.rev) + return false; + + if (subdir != other.subdir) + return false; + + return true; +} + std::optional parseFlakeRef( const std::string & uri, bool allowRelative) { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 6b47330a7..39e019dbd 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -182,6 +182,12 @@ struct FlakeRef return std::get_if(&data) && rev == Hash(rev->type); } + + /* Return true if 'other' is not less specific than 'this'. For + example, 'nixpkgs' contains 'nixpkgs/release-19.03', and both + 'nixpkgs' and 'nixpkgs/release-19.03' contain + 'nixpkgs/release-19.03/'. */ + bool contains(const FlakeRef & other) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); From 5573365dffc96df5fcb6898077aeff1cd5e2b711 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2019 20:15:42 +0200 Subject: [PATCH 0497/2548] nix flake check: Validate nixosConfigurations outputs --- src/nix/flake.cc | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 599bf12eb..2e352306e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -8,6 +8,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" +#include "attr-path.hh" #include #include @@ -341,6 +342,21 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON } }; + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking NixOS configuration '%s'", attrPath)); + Bindings & bindings(*state->allocBindings(0)); + auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v); + state->forceAttrs(*vToplevel, pos); + if (!state->isDerivation(*vToplevel)) + throw Error("attribute 'config.system.build.toplevel' is not a derivation"); + } catch (Error & e) { + e.addPrefix(fmt("while checking the NixOS configuration '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); @@ -407,6 +423,13 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON *attr.value, *attr.pos); } + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkNixOSConfiguration(name + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + else if (name == "hydraJobs") checkHydraJobs(name, vOutput, pos); From 99e8e58f2de9941353b47ed14fbe4ed76d635519 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 13:48:53 +0200 Subject: [PATCH 0498/2548] Shut up some warnings --- src/libstore/download.hh | 2 ++ src/libstore/fs-accessor.hh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/download.hh b/src/libstore/download.hh index abc4a828c..c5dd893b5 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -94,6 +94,8 @@ class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; From 68e0f23edc7622290809c81f5071c9f99aded459 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 14:29:49 +0200 Subject: [PATCH 0499/2548] Add flags to disallow dirty Git trees and to turn off warnings --- src/libexpr/eval.hh | 8 +++++++- src/libexpr/flake/flake.cc | 7 ++++--- src/libexpr/primops/fetchGit.cc | 6 +++++- src/libexpr/primops/fetchMercurial.cc | 6 +++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 16350a5bf..1cc3c8507 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -377,10 +377,16 @@ struct EvalSettings : Config "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; Setting traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; Setting flakeRegistry{this, "https://raw.githubusercontent.com/NixOS/flake-registry/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index d9bbf85c2..f18159c05 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -480,9 +480,10 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { - if (lockFile.isDirty()) - warn("will not write lock file of flake '%s' because it has a dirty input", topRef); - else { + if (lockFile.isDirty()) { + if (evalSettings.warnDirty) + warn("will not write lock file of flake '%s' because it has a dirty input", topRef); + } else { lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 48d84c4a1..1b55b6f32 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -47,7 +47,11 @@ GitInfo exportGit(ref store, std::string uri, /* This is an unclean working tree. So copy all tracked files. */ - warn("Git tree '%s' is dirty", uri); + if (!evalSettings.allowDirty) + throw Error("Git tree '%s' is dirty", uri); + + if (evalSettings.warnDirty) + warn("Git tree '%s' is dirty", uri); GitInfo gitInfo; gitInfo.ref = "HEAD"; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c791443c3..40082894f 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -36,7 +36,11 @@ HgInfo exportMercurial(ref store, const std::string & uri, /* This is an unclean working tree. So copy all tracked files. */ - printTalkative("copying unclean Mercurial working tree '%s'", uri); + if (!evalSettings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", uri); + + if (evalSettings.warnDirty) + warn("Mercurial tree '%s' is unclean", uri); HgInfo hgInfo; hgInfo.rev = "0000000000000000000000000000000000000000"; From 5961c9409739751caaea54540607703c53c9a37e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 14:46:37 +0200 Subject: [PATCH 0500/2548] Flake alias -> id --- src/libexpr/flake/flakeref.cc | 12 ++++++------ src/libexpr/flake/flakeref.hh | 13 ++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 4930d03ce..364a98358 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -20,7 +20,7 @@ const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + // "master/e72daba8250068216d79d2aeef40d4d95aff6666"). const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; // GitHub references. const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; @@ -43,7 +43,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) // FIXME: could combine this into one regex. static std::regex flakeRegex( - "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", + "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?", std::regex::ECMAScript); static std::regex githubRegex( @@ -90,8 +90,8 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) std::cmatch match; if (std::regex_match(uri.c_str(), match, flakeRegex)) { - IsAlias d; - d.alias = match[1]; + IsId d; + d.id = match[1]; if (match[2].matched) rev = Hash(match[2], htSHA1); else if (match[3].matched) { @@ -195,8 +195,8 @@ std::string FlakeRef::to_string() const string += value; // FIXME: escaping }; - if (auto refData = std::get_if(&data)) { - string = refData->alias; + if (auto refData = std::get_if(&data)) { + string = refData->id; if (ref) string += '/' + *ref; if (rev) string += '/' + rev->gitRev(); } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 39e019dbd..addf5449f 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -101,16 +101,15 @@ namespace nix { */ typedef std::string FlakeId; -typedef std::string FlakeAlias; typedef std::string FlakeUri; struct FlakeRef { - struct IsAlias + struct IsId { - FlakeAlias alias; - bool operator<(const IsAlias & b) const { return alias < b.alias; }; - bool operator==(const IsAlias & b) const { return alias == b.alias; }; + FlakeId id; + bool operator<(const IsId & b) const { return id < b.id; }; + bool operator==(const IsId & b) const { return id == b.id; }; }; struct IsGitHub { @@ -140,7 +139,7 @@ struct FlakeRef // Git, Tarball - std::variant data; + std::variant data; std::optional ref; std::optional rev; @@ -168,7 +167,7 @@ struct FlakeRef a flake ID, which requires a lookup in the flake registry. */ bool isDirect() const { - return !std::get_if(&data); + return !std::get_if(&data); } /* Check whether this is an "immutable" flake reference, that is, From 5a0e98d1e5697d1d28131e3fd893282be660e008 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 16:01:40 +0200 Subject: [PATCH 0501/2548] Use '#' instead of ':' to separate flakeref and attrpath This is less ambiguous. --- src/nix/installables.cc | 22 ++++++++----------- tests/flakes.sh | 48 ++++++++++++++++++++--------------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index a4726a59e..987b744fe 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -404,7 +404,7 @@ std::vector> SourceExprCommand::parseInstallables( for (auto & s : ss) { - size_t colon; + size_t hash; std::optional storePath; if (s.compare(0, 1, "(") == 0) @@ -417,8 +417,14 @@ std::vector> SourceExprCommand::parseInstallables( Strings{"legacyPackages." + std::string(s, 8)})); } - else { + else if ((hash = s.rfind('#')) != std::string::npos) + result.push_back(std::make_shared( + *this, + FlakeRef(std::string(s, 0, hash), true), + std::string(s, hash + 1), + getDefaultFlakeAttrPathPrefixes())); + else { std::exception_ptr flakeEx; try { @@ -434,17 +440,7 @@ std::vector> SourceExprCommand::parseInstallables( } catch (BadFlakeRef &) { } - if ((colon = s.rfind(':')) != std::string::npos) { - auto flakeRef = std::string(s, 0, colon); - auto attrPath = std::string(s, colon + 1); - result.push_back(std::make_shared( - *this, - FlakeRef(flakeRef, true), - attrPath, - getDefaultFlakeAttrPathPrefixes())); - } - - else if (s.find('/') != std::string::npos && (storePath = follow(s))) + if (s.find('/') != std::string::npos && (storePath = follow(s))) result.push_back(std::make_shared(*storePath)); else { diff --git a/tests/flakes.sh b/tests/flakes.sh index f3baf8cef..a91a0c37c 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -126,7 +126,7 @@ json=$(nix flake info --flake-registry $registry flake1 --json | jq .) [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] # Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result --flake-registry $registry flake1:foo +nix build -o $TEST_ROOT/result --flake-registry $registry flake1#foo [[ -e $TEST_ROOT/result/hello ]] # Test defaultPackage. @@ -144,33 +144,33 @@ nix path-info $flake1Dir/result (! nix eval "(builtins.getFlake "$flake2Dir")") # But should succeed in impure mode. -nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar --impure +nix build -o $TEST_ROOT/result --flake-registry $registry flake2#bar --impure # Test automatic lock file generation. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir#bar [[ -e $flake2Dir/flake.lock ]] git -C $flake2Dir commit flake.lock -m 'Add flake.lock' # Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry $registry $flake2Dir#bar [[ -z $(git -C $flake2Dir diff master) ]] # Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir:bar --tarball-ttl 0 +nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --tarball-ttl 0 # Updating the flake should not change the lockfile. nix flake update --flake-registry $registry $flake2Dir [[ -z $(git -C $flake2Dir diff master) ]] # Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result --flake-registry $registry flake2:bar +nix build -o $TEST_ROOT/result --flake-registry $registry flake2#bar # Or without a registry. # FIXME: shouldn't need '--flake-registry /no-registry'? -nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir:bar --tarball-ttl 0 +nix build -o $TEST_ROOT/result --flake-registry /no-registry file://$flake2Dir#bar --tarball-ttl 0 # Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:xyzzy +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir#xyzzy # Add dependency to flake3. rm $flake3Dir/flake.nix @@ -192,7 +192,7 @@ git -C $flake3Dir add flake.nix git -C $flake3Dir commit -m 'Update flake.nix' # Check whether `nix build` works with an incomplete lockfile -nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:"sth sth" +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir#"sth sth" # Check whether it saved the lockfile [[ ! (-z $(git -C $flake3Dir diff master)) ]] @@ -203,7 +203,7 @@ git -C $flake3Dir commit -m 'Add lockfile' # Unsupported editions should be an error. sed -i $flake3Dir/flake.nix -e s/201909/201912/ -nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth 2>&1 | grep 'unsupported edition' +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir#sth 2>&1 | grep 'unsupported edition' # Test whether registry caching works. nix flake list --flake-registry file://$registry | grep -q flake3 @@ -214,12 +214,12 @@ mv $registry.tmp $registry # Test whether flakes are registered as GC roots for offline use. # FIXME: use tarballs rather than git. rm -rf $TEST_HOME/.cache -nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir#bar mv $flake1Dir $flake1Dir.tmp mv $flake2Dir $flake2Dir.tmp nix-store --gc -nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar -nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir:bar --tarball-ttl 0 +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir#bar +nix build -o $TEST_ROOT/result --flake-registry file://$registry file://$flake2Dir#bar --tarball-ttl 0 mv $flake1Dir.tmp $flake1Dir mv $flake2Dir.tmp $flake2Dir @@ -264,30 +264,30 @@ git -C $flake3Dir commit -m 'Add nonFlakeInputs' # Check whether `nix build` works with a lockfile which is missing a # nonFlakeInputs. -nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir:sth +nix build -o $TEST_ROOT/result --flake-registry $registry $flake3Dir#sth git -C $flake3Dir add flake.lock git -C $flake3Dir commit -m 'Update nonFlakeInputs' -nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord +nix build -o $TEST_ROOT/result --flake-registry $registry flake3#fnord [[ $(cat $TEST_ROOT/result) = FNORD ]] -# Check whether flake input fetching is lazy: flake3:sth does not +# Check whether flake input fetching is lazy: flake3#sth does not # depend on flake2, so this shouldn't fail. rm -rf $TEST_HOME/.cache clearStore mv $flake2Dir $flake2Dir.tmp mv $nonFlakeDir $nonFlakeDir.tmp -nix build -o $TEST_ROOT/result --flake-registry $registry flake3:sth -(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy) -(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3:fnord) +nix build -o $TEST_ROOT/result --flake-registry $registry flake3#sth +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3#xyzzy) +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake3#fnord) mv $flake2Dir.tmp $flake2Dir mv $nonFlakeDir.tmp $nonFlakeDir -nix build -o $TEST_ROOT/result --flake-registry $registry flake3:xyzzy flake3:fnord +nix build -o $TEST_ROOT/result --flake-registry $registry flake3#xyzzy flake3#fnord # Test doing multiple `lookupFlake`s -nix build -o $TEST_ROOT/result --flake-registry $registry flake4:xyzzy +nix build -o $TEST_ROOT/result --flake-registry $registry flake4#xyzzy # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore git -C $flake3Dir checkout -b removeXyzzy @@ -325,11 +325,11 @@ git -C $flake3Dir commit -m 'Remove packages.xyzzy' git -C $flake3Dir checkout master # Test whether fuzzy-matching works for IsAlias -(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:xyzzy) +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy#xyzzy) # Test whether fuzzy-matching works for IsGit -(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:xyzzy) -nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy:sth +(! nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy#xyzzy) +nix build -o $TEST_ROOT/result --flake-registry $registry flake4/removeXyzzy#sth # Testing the nix CLI nix flake add --flake-registry $registry flake1 flake3 From 14d3f450098e0f4a9b8d538545f1d1bd1decdab3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 16:06:49 +0200 Subject: [PATCH 0502/2548] Simplify --- src/nix/installables.cc | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 987b744fe..85005cc95 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -425,29 +425,15 @@ std::vector> SourceExprCommand::parseInstallables( getDefaultFlakeAttrPathPrefixes())); else { - std::exception_ptr flakeEx; - try { auto flakeRef = FlakeRef(s, true); result.push_back(std::make_shared( *this, std::move(flakeRef), getDefaultFlakeAttrPaths())); - continue; - } catch (MissingFlake &) { - /* 's' could be parsed as a flakeref, but it - references a local path that is not a flake. So - take note of that. */ - flakeEx = std::current_exception(); - } catch (BadFlakeRef &) { - } - - if (s.find('/') != std::string::npos && (storePath = follow(s))) - result.push_back(std::make_shared(*storePath)); - - else { - if (flakeEx) - std::rethrow_exception(flakeEx); + } catch (...) { + if (s.find('/') != std::string::npos && (storePath = follow(s))) + result.push_back(std::make_shared(*storePath)); else - throw Error("unsupported argument '%s'", s); + throw; } } } From c6a542f22a165f711fc0730bfcfbd936ad1c3dfb Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sat, 21 Sep 2019 18:53:15 -0400 Subject: [PATCH 0503/2548] Fix spelling in comment --- src/libstore/build.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ab725e8e9..9a6729f9e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2364,7 +2364,7 @@ void DerivationGoal::startBuilder() child = clone(childEntry, stack + stackSize, flags, this); } if (child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unpriveleged + /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. */ @@ -2410,8 +2410,7 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); - /* Signal the builder that we've updated its user - namespace. */ + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; From e63c9e73e3e5d1f31fa5065c9ff59f442dd07d0e Mon Sep 17 00:00:00 2001 From: zimbatm Date: Sun, 22 Sep 2019 10:57:20 +0000 Subject: [PATCH 0504/2548] libstore: don't forward --show-trace --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c2e23f9c..76107db54 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -198,6 +198,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.maxSilentTime.name); overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; From 893be6f5e36abb58bbaa9c49055a5218114dd514 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Sep 2019 21:29:33 +0200 Subject: [PATCH 0505/2548] Don't catch exceptions by value --- src/libexpr/json-to-value.cc | 4 ++-- src/libexpr/primops/fetchGit.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/nix-build/nix-build.cc | 2 +- src/nix/edit.cc | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 3f6017957..8bae986f9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkFloat(v, stod(tmp_number)); else mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range e) { + } catch (std::out_of_range & e) { throw JSONParseError("out-of-range JSON number"); } } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 1b55b6f32..64f138195 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -37,7 +37,7 @@ GitInfo exportGit(ref store, std::string uri, try { runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError e) { + } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; clean = false; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1062df781..307c00dbb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -872,8 +872,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { - } catch (SubstituterDisabled) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) printError(e.what()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index d4749f8fd..605802f72 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -280,7 +280,7 @@ static void _main(int argc, char * * argv) auto absolute = i; try { absolute = canonPath(absPath(i), true); - } catch (Error e) {}; + } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c62b35c46..632a99d11 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -50,7 +50,7 @@ struct CmdEdit : InstallableCommand int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } From 382aa05ff71b61379f5c2792eaf517bdf4a5c5bf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Sep 2019 21:53:01 +0200 Subject: [PATCH 0506/2548] nix flake info --json: Get rid of duplicate getFlake() call Also fix some gcc warnings. --- src/nix/flake.cc | 19 ++++++++++--------- src/nix/installables.cc | 7 ++++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 2e352306e..d0135143c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -174,11 +174,12 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, { state.forceAttrs(vFlake); - auto vOutputs = (*vFlake.attrs->get(state.symbols.create("outputs")))->value; + auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + assert(aOutputs); - state.forceAttrs(*vOutputs); + state.forceAttrs(*(*aOutputs)->value); - for (auto & attr : *vOutputs->attrs) + for (auto & attr : *((*aOutputs)->value->attrs)) callback(attr.name, *attr.value, *attr.pos); } @@ -191,15 +192,12 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON void run(nix::ref store) override { - auto flake = getFlake(); - stopProgressBar(); - if (json) { - auto json = flakeToJson(flake); - auto state = getEvalState(); auto flake = resolveFlake(); + auto json = flakeToJson(flake.flake); + auto vFlake = state->allocValue(); flake::callFlake(*state, flake, *vFlake); @@ -222,8 +220,11 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON json["outputs"] = std::move(outputs); std::cout << json.dump() << std::endl; - } else + } else { + auto flake = getFlake(); + stopProgressBar(); printFlakeInfo(flake); + } } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 85005cc95..867133653 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -291,11 +291,12 @@ struct InstallableFlake : InstallableValue makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); - auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); - state.forceValue(*vOutputs); + state.forceValue(*(*aOutputs)->value); - return vOutputs; + return (*aOutputs)->value; } std::vector toDerivations() override From bd79c1f6f6391786772a8a79962abe22f374cca4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Sep 2019 21:29:33 +0200 Subject: [PATCH 0507/2548] Don't catch exceptions by value (cherry picked from commit 893be6f5e36abb58bbaa9c49055a5218114dd514) --- src/libexpr/json-to-value.cc | 4 ++-- src/libexpr/primops/fetchGit.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/nix-build/nix-build.cc | 2 +- src/nix/edit.cc | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 3f6017957..8bae986f9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkFloat(v, stod(tmp_number)); else mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range e) { + } catch (std::out_of_range & e) { throw JSONParseError("out-of-range JSON number"); } } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 6229fef8d..90f600284 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -38,7 +38,7 @@ GitInfo exportGit(ref store, const std::string & uri, try { runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError e) { + } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; clean = false; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2fcf08491..a2af51d0e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -879,8 +879,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { - } catch (SubstituterDisabled) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) printError(e.what()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c6a4d4166..33ad28704 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -280,7 +280,7 @@ static void _main(int argc, char * * argv) auto absolute = i; try { absolute = canonPath(absPath(i), true); - } catch (Error e) {}; + } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d..a6169f118 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -55,7 +55,7 @@ struct CmdEdit : InstallableCommand int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } From 3a022d45993b6fa8c7bf03517a3a3d1a2ab15f4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 20 Sep 2019 13:48:53 +0200 Subject: [PATCH 0508/2548] Shut up some warnings (cherry picked from commit 99e8e58f2de9941353b47ed14fbe4ed76d635519) --- src/libstore/download.hh | 2 ++ src/libstore/fs-accessor.hh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c68381846..68565bf46 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -91,6 +91,8 @@ class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; From 10bfc5c0d09d5508e3dab4c32f3368caeb5f7f56 Mon Sep 17 00:00:00 2001 From: Joseph Lucas Date: Mon, 23 Sep 2019 13:18:59 +0000 Subject: [PATCH 0509/2548] Update garbage-collection.xml readability 1. remove a typo space 2. Simplify negative style by using affirmative style --- doc/manual/packages/garbage-collection.xml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml index a1b0ef22a..b506f22b0 100644 --- a/doc/manual/packages/garbage-collection.xml +++ b/doc/manual/packages/garbage-collection.xml @@ -52,12 +52,13 @@ garbage collector as follows: $ nix-store --gc -The behaviour of the gargage collector is affected by the keep- -derivations (default: true) and keep-outputs +The behaviour of the gargage collector is affected by the +keep-derivations (default: true) and keep-outputs (default: false) options in the Nix configuration file. The defaults will ensure -that all derivations that are not build-time dependencies of garbage collector roots -will be collected but that all output paths that are not runtime dependencies -will be collected. (This is usually what you want, but while you are developing +that all derivations that are build-time dependencies of garbage collector roots +will be kept and that all output paths that are runtime dependencies +will be kept as well. All other derivations or paths will be collected. +(This is usually what you want, but while you are developing it may make sense to keep outputs to ensure that rebuild times are quick.) If you are feeling uncertain, you can also first view what files would From c32bba748902a8603036744d4ebccc134f02958c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Sep 2019 17:28:18 +0200 Subject: [PATCH 0510/2548] Shut up some warnings --- src/libstore/nar-info-disk-cache.hh | 2 ++ src/libstore/sqlite.hh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 88d909732..0646d5a94 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,6 +10,8 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; + virtual ~NarInfoDiskCache() { } + virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 78e53fa32..0f46f6a07 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -5,8 +5,8 @@ #include "types.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { From 454e3a541a05a9b422ae5cd0911316f589f02506 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Sep 2019 17:51:51 +0200 Subject: [PATCH 0511/2548] Fix sorting of non-flake input attributes --- src/libexpr/flake/flake.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index f18159c05..2061a34a4 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -146,7 +146,7 @@ static FlakeRef lookupInRefMap( return flakeRef; } -static SourceInfo fetchFlake(EvalState & state, const FlakeRef & resolvedRef) +static SourceInfo fetchInput(EvalState & state, const FlakeRef & resolvedRef) { assert(resolvedRef.isDirect()); @@ -231,7 +231,7 @@ static Flake getFlake(EvalState & state, const FlakeRef & originalRef, maybeLookupFlake(state, lookupInRefMap(refMap, originalRef), allowLookup)); - SourceInfo sourceInfo = fetchFlake(state, flakeRef); + SourceInfo sourceInfo = fetchInput(state, flakeRef); debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; @@ -352,7 +352,7 @@ static SourceInfo getNonFlake(EvalState & state, const FlakeRef & originalRef, maybeLookupFlake(state, lookupInRefMap(refMap, originalRef), allowLookup)); - auto sourceInfo = fetchFlake(state, flakeRef); + auto sourceInfo = fetchInput(state, flakeRef); debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); FlakeRef resolvedRef = sourceInfo.resolvedRef; @@ -566,6 +566,8 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V sourceInfo.storePath, {sourceInfo.storePath}); emitSourceInfoAttrs(state, sourceInfo, v); + + v.attrs->sort(); } } From 15b888c9a589e71a6e3b9bc2cfcb3679f90fbf70 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Sep 2019 15:31:09 +0200 Subject: [PATCH 0512/2548] cmatch -> smatch --- src/libexpr/flake/flakeref.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 364a98358..8e90e5989 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -88,8 +88,8 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) return true; }; - std::cmatch match; - if (std::regex_match(uri.c_str(), match, flakeRegex)) { + std::smatch match; + if (std::regex_match(uri, match, flakeRegex)) { IsId d; d.id = match[1]; if (match[2].matched) @@ -102,7 +102,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) data = d; } - else if (std::regex_match(uri.c_str(), match, githubRegex)) { + else if (std::regex_match(uri, match, githubRegex)) { IsGitHub d; d.owner = match[1]; d.repo = match[2]; @@ -120,7 +120,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) data = d; } - else if (std::regex_match(uri.c_str(), match, uriRegex)) { + else if (std::regex_match(uri, match, uriRegex)) { auto & scheme = match[2]; if (scheme == "git" || scheme == "git+http" || From 9b9de3a5e321c764ec018b32c1f949a49b0c69ef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Sep 2019 17:01:25 +0200 Subject: [PATCH 0513/2548] nix dev-shell: Improve environment handling Only variables that were marked as exported are exported in the dev shell. Also, we no longer try to parse the function section of the env file, fixing $ nix dev-shell error: shell environment '/nix/store/h7ama3kahb8lypf4nvjx34z06g9ncw4h-nixops-1.7pre20190926.4c7acbb-env' has unexpected line '/^[a-z]?"""/ {' --- src/nix/shell.cc | 110 +++++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index a3827c297..db7106793 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -7,55 +7,76 @@ #include "affinity.hh" #include "progress-bar.hh" +#include + using namespace nix; +struct Var +{ + bool exported; + std::string value; // quoted string or array +}; + struct BuildEnvironment { - // FIXME: figure out which vars should be exported. - std::map env; - std::map functions; + std::map env; + std::string bashFunctions; }; BuildEnvironment readEnvironment(const Path & path) { BuildEnvironment res; - auto lines = tokenizeString(readFile(path), "\n"); + std::set exported; - auto getLine = - [&]() { - if (lines.empty()) - throw Error("shell environment '%s' ends unexpectedly", path); - auto line = lines.front(); - lines.pop_front(); - return line; - }; + auto file = readFile(path); + //auto file = readFile("/tmp/x"); - while (!lines.empty()) { - auto line = getLine(); + auto pos = file.cbegin(); - auto eq = line.find('='); - if (eq != std::string::npos) { - std::string name(line, 0, eq); - std::string value(line, eq + 1); - // FIXME: parse arrays - res.env.insert({name, value}); + static std::string varNameRegex = + R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + + static std::regex declareRegex( + "^declare -x (" + varNameRegex + ")" + + R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + + static std::string simpleStringRegex = + R"re((?:[a-zA-Z0-9_/:\.\-1\+]*))re"; + + static std::string quotedStringRegex = + R"re((?:\$?'[^']*'))re"; + + static std::string arrayRegex = + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")*\)))re"; + + static std::regex varRegex( + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + arrayRegex + ")\n"); + + static std::regex functionRegex( + "^" + varNameRegex + " \\(\\) *\n"); + + while (pos != file.end()) { + + std::smatch match; + + if (std::regex_search(pos, file.cend(), match, declareRegex)) { + pos = match[0].second; + exported.insert(match[1]); } - else if (hasSuffix(line, " () ")) { - std::string name(line, 0, line.size() - 4); - // FIXME: validate name - auto l = getLine(); - if (l != "{ ") throw Error("shell environment '%s' has unexpected line '%s'", path, l); - std::string body; - while ((l = getLine()) != "}") { - body += l; - body += '\n'; - } - res.functions.insert({name, body}); + else if (std::regex_search(pos, file.cend(), match, varRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { (bool) exported.count(match[1]), match[2] }}); } - else throw Error("shell environment '%s' has unexpected line '%s'", path, line); + else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + res.bashFunctions = std::string(pos, file.cend()); + break; + } + + else throw Error("shell environment '%s' has unexpected line '%s'", + path, file.substr(pos - file.cbegin(), 60)); } return res; @@ -72,7 +93,16 @@ Path getDerivationEnvironment(ref store, Derivation drv) if (builder != "bash") throw Error("'nix shell' only works on derivations that use 'bash' as their builder"); - drv.args = {"-c", "set -e; export IN_NIX_SHELL=impure; export dontAddDisableDepTrack=1; if [[ -n $stdenv ]]; then source $stdenv/setup; fi; set > $out"}; + drv.args = { + "-c", + "set -e; " + "export IN_NIX_SHELL=impure; " + "export dontAddDisableDepTrack=1; " + "if [[ -n $stdenv ]]; then " + " source $stdenv/setup; " + "fi; " + "export > $out; " + "set >> $out "}; /* Remove derivation checks. */ drv.env.erase("allowedReferences"); @@ -146,18 +176,16 @@ struct Common : InstallableCommand, MixProfile out << "nix_saved_PATH=\"$PATH\"\n"; for (auto & i : buildEnvironment.env) { - // FIXME: shellEscape - // FIXME: figure out what to export - // FIXME: handle arrays - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) - out << fmt("export %s=%s\n", i.first, i.second); + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } } out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - for (auto & i : buildEnvironment.functions) { - out << fmt("%s () {\n%s\n}\n", i.first, i.second); - } + out << buildEnvironment.bashFunctions << "\n"; // FIXME: set outputs From 168a8879165dd0deab1a93d343a6003146f37031 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 12 Jun 2019 10:34:13 +0200 Subject: [PATCH 0514/2548] Fix fetchTarball with chroot stores Fixes #2405. --- src/libexpr/primops.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 350dba474..d4c60f870 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2089,12 +2089,12 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, request).path; + auto res = getDownloader()->downloadCached(state.store, request); if (state.allowedPaths) - state.allowedPaths->insert(res); + state.allowedPaths->insert(res.path); - mkString(v, res, PathSet({res})); + mkString(v, res.storePath, PathSet({res.storePath})); } From 74b4737d8f0e1922ef5314a158271acf81cd79f8 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:07:07 -0400 Subject: [PATCH 0515/2548] Add libatomic for 32-bit ARM Fixes #3113 --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index a52830b38..96a2065b1 100644 --- a/configure.ac +++ b/configure.ac @@ -157,6 +157,11 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" +# Boost atomic needs GCC libatomic on 32-bit ARM +case "$host_cpu" in + armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" +esac + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From b1c34152feac76c9baded58732068541b7a586f1 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 1 Oct 2019 21:22:18 -0400 Subject: [PATCH 0516/2548] Use more robust test for libatomics Taken from Mesa configure script: https://github.com/mesa3d/mesa/blob/17.2/configure.ac#L405-L427 --- configure.ac | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 96a2065b1..ebe6d4267 100644 --- a/configure.ac +++ b/configure.ac @@ -157,11 +157,18 @@ AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix # ends up with LDFLAGS being empty, so we set it afterwards. LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" -# Boost atomic needs GCC libatomic on 32-bit ARM -case "$host_cpu" in - armv5*|armv6*|armv7*) LIBS="-latomic $LIBS" -esac - +# On some platforms, new-style atomics need a helper library +AC_MSG_CHECKING(whether -latomic is needed) +AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#include +uint64_t v; +int main() { + return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE); +}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) +AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) +if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then + LIBS="-latomic $LIBS" +fi # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) From 780c1a8f271113cea7d70d57f44afce5da5928d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Oct 2019 10:52:56 +0200 Subject: [PATCH 0517/2548] nix dev-shell: Ignore $NIX_LOG_FD --- src/nix/shell.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index db7106793..5e1d53aff 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -157,6 +157,7 @@ struct Common : InstallableCommand, MixProfile "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", + "NIX_LOG_FD", "PPID", "PWD", "SHELLOPTS", From 15e70c662e86c42658cdbc76b54da0f3f2be3c30 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Oct 2019 16:26:15 +0200 Subject: [PATCH 0518/2548] Fix indentation --- src/nix-build/nix-build.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 33ad28704..2b36846cd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -282,12 +282,12 @@ static void _main(int argc, char * * argv) absolute = canonPath(absPath(i), true); } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(*state, store, absolute)); - else - /* If we're in a #! script, interpret filenames - relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); } } From 90d6018509e60fd07d93ddefbf6f983a72d4b587 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Oct 2019 22:08:19 +0200 Subject: [PATCH 0519/2548] Fix aborts when using builtins.getFlake In that case, 'self' could refer to a value on the stack, so accessing 'self.rev' would abort. --- src/libexpr/flake/flake.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2061a34a4..9e260263c 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -574,8 +574,9 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V void callFlake(EvalState & state, const Flake & flake, const LockedInputs & lockedInputs, - Value & vRes) + Value & vResFinal) { + auto & vRes = *state.allocValue(); auto & vInputs = *state.allocValue(); state.mkAttrs(vInputs, flake.inputs.size() + 1); @@ -617,6 +618,8 @@ void callFlake(EvalState & state, state.callFunction(vCall, *flake.vOutputs, vCall2, noPos); state.callFunction(vCall2, vInputs, vCall3, noPos); state.callFunction(vCall3, vSourceInfo, vRes, noPos); + + vResFinal = vRes; } void callFlake(EvalState & state, From 204291f0598c43b10cf20a89b49fc63624d78c7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 10:45:33 +0200 Subject: [PATCH 0520/2548] Merge release.nix, shell.nix and release-common.nix into flake.nix Also provide a Nixpkgs overlay, memoize Nixpkgs evaluation and fit the githubFlakes test. --- flake.lock | 6 +- flake.nix | 468 +++++++++++++++++++++++++++++++++++-- release-common.nix | 80 ------- release.nix | 396 ------------------------------- shell.nix | 28 --- tests/github-flakes.nix | 12 +- tests/nix-copy-closure.nix | 9 +- tests/remote-builds.nix | 9 +- tests/setuid.nix | 8 +- 9 files changed, 476 insertions(+), 540 deletions(-) delete mode 100644 release-common.nix delete mode 100644 release.nix delete mode 100644 shell.nix diff --git a/flake.lock b/flake.lock index 70a433b26..05e3a6e25 100644 --- a/flake.lock +++ b/flake.lock @@ -2,9 +2,9 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", - "originalUri": "nixpkgs", - "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" + "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", + "originalUri": "nixpkgs/release-19.03", + "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" } }, "version": 3 diff --git a/flake.nix b/flake.nix index f0d928494..220b6d8c5 100644 --- a/flake.nix +++ b/flake.nix @@ -3,27 +3,461 @@ edition = 201909; - outputs = { self, nixpkgs }: rec { + inputs.nixpkgs.uri = "nixpkgs/release-19.03"; - hydraJobs = import ./release.nix { - nix = self; - nixpkgs = nixpkgs; - }; + outputs = { self, nixpkgs }: - checks = { - binaryTarball = hydraJobs.binaryTarball.x86_64-linux; - perlBindings = hydraJobs.perlBindings.x86_64-linux; - }; + let - packages = { - nix = hydraJobs.build.x86_64-linux; - nix-perl-bindings = hydraJobs.perlBindings.x86_64-linux; - }; + officialRelease = false; - defaultPackage = packages.nix; + systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + + # Memoize nixpkgs for different platforms for efficiency. + nixpkgsFor = nixpkgs.lib.genAttrs systems (system: + import nixpkgs { + inherit system; + overlays = [ self.overlay ]; + } + ); + + commonDeps = pkgs: with pkgs; rec { + # Use "busybox-sandbox-shell" if present, + # if not (legacy) fallback and hope it's sufficient. + sh = pkgs.busybox-sandbox-shell or (busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + configureFlags = + [ + "--enable-gc" + ] ++ lib.optionals stdenv.isLinux [ + "--with-sandbox-shell=${sh}/bin/busybox" + ]; + + tarballDeps = + [ bison + flex + libxml2 + libxslt + docbook5 + docbook_xsl_ns + autoconf-archive + autoreconfHook + ]; + + buildDeps = + [ curl + bzip2 xz brotli editline + openssl pkgconfig sqlite boehmgc + boost + + # Tests + git + mercurial + jq + ] + ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }); + + perlDeps = + [ perl + perlPackages.DBDSQLite + ]; + }; + + in { + + # A Nixpkgs overlay that overrides the 'nix' and 'nix-perl' + # packages. + overlay = final: prev: { + + nix = with final; with commonDeps pkgs; releaseTools.nixBuild { + name = "nix"; + src = self.hydraJobs.tarball; + + buildInputs = buildDeps; + + preConfigure = + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + '' + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ''; + + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d"; + + installFlags = "sysconfdir=$(out)/etc"; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + }; + + nix-perl = with final; releaseTools.nixBuild { + name = "nix-perl"; + src = self.hydraJobs.tarball; + + buildInputs = + [ nix curl bzip2 xz pkgconfig pkgs.perl boost ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + + configureFlags = '' + --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} + --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} + ''; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; + }; + + }; + + hydraJobs = { + + # Source tarball. + tarball = + with nixpkgsFor.x86_64-linux; + with commonDeps pkgs; + + releaseTools.sourceTarball { + name = "nix-tarball"; + version = builtins.readFile ./.version; + versionSuffix = if officialRelease then "" else + "pre${builtins.substring 0 8 self.lastModified}_${self.shortRev}"; + src = self; + inherit officialRelease; + + buildInputs = tarballDeps ++ buildDeps; + + configureFlags = "--enable-gc"; + + postUnpack = '' + (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files + cat $sourceRoot/.dist-files + ''; + + preConfigure = '' + (cd perl ; autoreconf --install --force --verbose) + # TeX needs a writable font cache. + export VARTEXFONTS=$TMPDIR/texfonts + ''; + + distPhase = + '' + runHook preDist + make dist + mkdir -p $out/tarballs + cp *.tar.* $out/tarballs + ''; + + preDist = '' + make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk + echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products + ''; + }; + + # Binary package for various platforms. + build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix); + + # Perl bindings for various platforms. + perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix-perl); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = nixpkgs.lib.genAttrs systems (system: + + with nixpkgsFor.${system}; + + let + version = nix.src.version; + installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; }; + in + + runCommand "nix-binary-tarball-${version}" + { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = + with nixpkgsFor.x86_64-linux; + runCommand "installer-script" + { buildInputs = [ nix ]; + } + '' + mkdir -p $out/nix-support + + substitute ${./scripts/install.in} $out/install \ + ${pkgs.lib.concatMapStrings + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ") + [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] + } \ + --replace '@nixVersion@' ${nix.src.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products + ''; + + # Line coverage analysis. + coverage = + with nixpkgsFor.x86_64-linux; + with commonDeps pkgs; + + releaseTools.coverageAnalysis { + name = "nix-build"; + src = self.hydraJobs.tarball; + + buildInputs = buildDeps; + + dontInstall = false; + + doInstallCheck = true; + + lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ]; + + # We call `dot', and even though we just use it to + # syntax-check generated dot files, it still requires some + # fonts. So provide those. + FONTCONFIG_FILE = texFunctions.fontsConf; + }; + + # System tests. + tests.remoteBuilds = import ./tests/remote-builds.nix { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }; + + tests.nix-copy-closure = import ./tests/nix-copy-closure.nix { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }; + + tests.githubFlakes = (import ./tests/github-flakes.nix rec { + system = "x86_64-linux"; + inherit nixpkgs; + inherit (self) overlay; + }); + + tests.setuid = nixpkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs system; + inherit (self) overlay; + }); + + # Test whether the binary tarball works in an Ubuntu system. + tests.binaryTarball = + with nixpkgsFor.x86_64-linux; + vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" + { diskImage = vmTools.diskImages.ubuntu1204x86_64; + } + '' + set -x + useradd -m alice + su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*' + mkdir /dest-nix + mount -o bind /dest-nix /nix # Provide a writable /nix. + chown alice /nix + su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' + su - alice -c 'nix-store --verify' + su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}' + + # Check whether 'nix upgrade-nix' works. + cat > /tmp/paths.nix < $sourceRoot/.dist-files - cat $sourceRoot/.dist-files - ''; - - preConfigure = '' - (cd perl ; autoreconf --install --force --verbose) - # TeX needs a writable font cache. - export VARTEXFONTS=$TMPDIR/texfonts - ''; - - distPhase = - '' - runHook preDist - make dist - mkdir -p $out/tarballs - cp *.tar.* $out/tarballs - ''; - - preDist = '' - make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk - echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products - ''; - }; - - - build = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - releaseTools.nixBuild { - name = "nix"; - src = tarball; - - buildInputs = buildDeps; - - preConfigure = - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - '' - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ]; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d"; - - installFlags = "sysconfdir=$(out)/etc"; - - doInstallCheck = true; - installCheckFlags = "sysconfdir=$(out)/etc"; - }); - - - perlBindings = pkgs.lib.genAttrs systems (system: - - let pkgs = import nixpkgs { inherit system; }; in with pkgs; - - releaseTools.nixBuild { - name = "nix-perl"; - src = tarball; - - buildInputs = - [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; - - configureFlags = '' - --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - ''; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); - - - binaryTarball = pkgs.lib.genAttrs systems (system: - - with import nixpkgs { inherit system; }; - - let - toplevel = builtins.getAttr system jobs.build; - version = toplevel.src.version; - installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; }; - in - - runCommand "nix-binary-tarball-${version}" - { nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${toplevel} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''); - - - coverage = - with pkgs; - - with import ./release-common.nix { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-build"; - src = tarball; - - buildInputs = buildDeps; - - dontInstall = false; - - doInstallCheck = true; - - lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ]; - - # We call `dot', and even though we just use it to - # syntax-check generated dot files, it still requires some - # fonts. So provide those. - FONTCONFIG_FILE = texFunctions.fontsConf; - }; - - - #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; - - - #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; - #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; - - #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; - #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; - - - # System tests. - tests.remoteBuilds = (import ./tests/remote-builds.nix rec { - inherit nixpkgs; - nix = build.x86_64-linux; system = "x86_64-linux"; - }); - - tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec { - inherit nixpkgs; - nix = build.x86_64-linux; system = "x86_64-linux"; - }); - - tests.githubFlakes = (import ./tests/github-flakes.nix rec { - inherit nixpkgs; - nix = build.x86_64-linux; system = "x86_64-linux"; - }); - - tests.setuid = pkgs.lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: - import ./tests/setuid.nix rec { - inherit nixpkgs; - nix = build.${system}; inherit system; - }); - - tests.binaryTarball = - with import nixpkgs { system = "x86_64-linux"; }; - vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" - { diskImage = vmTools.diskImages.ubuntu1204x86_64; - } - '' - set -x - useradd -m alice - su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' - mkdir /dest-nix - mount -o bind /dest-nix /nix # Provide a writable /nix. - chown alice /nix - su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' - su - alice -c 'nix-store --verify' - su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' - - # Check whether 'nix upgrade-nix' works. - cat > /tmp/paths.nix <> $out/nix-support/hydra-build-products - ''; - - - # Aggregate job containing the release-critical jobs. - release = pkgs.releaseTools.aggregate { - name = "nix-${tarball.version}"; - meta.description = "Release-critical builds"; - constituents = - [ tarball - build.i686-linux - build.x86_64-darwin - build.x86_64-linux - build.aarch64-linux - binaryTarball.i686-linux - binaryTarball.x86_64-darwin - binaryTarball.x86_64-linux - binaryTarball.aarch64-linux - tests.remoteBuilds - tests.nix-copy-closure - tests.binaryTarball - tests.evalNixpkgs - tests.evalNixOS - installerScript - ]; - }; - - }; - - - makeRPM_i686 = makeRPM "i686-linux"; - makeRPM_x86_64 = makeRPM "x86_64-linux"; - - makeRPM = - system: diskImageFun: extraPackages: - - with import nixpkgs { inherit system; }; - - releaseTools.rpmBuild rec { - name = "nix-rpm"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ] - ++ extraPackages; }; - # At most 2047MB can be simulated in qemu-system-i386 - memSize = 2047; - meta.schedulingPriority = 50; - postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; - #enableParallelBuilding = true; - }; - - - makeDeb_i686 = makeDeb "i686-linux"; - makeDeb_x86_64 = makeDeb "x86_64-linux"; - - makeDeb = - system: diskImageFun: extraPackages: extraDebPackages: - - with import nixpkgs { inherit system; }; - - releaseTools.debBuild { - name = "nix-deb"; - src = jobs.tarball; - diskImage = (diskImageFun vmTools.diskImageFuns) - { extraPackages = - [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ] - ++ extraPackages; }; - memSize = 2047; - meta.schedulingPriority = 50; - postInstall = "make installcheck"; - configureFlags = "--sysconfdir=/etc"; - debRequires = - [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ] - ++ extraDebPackages; - debMaintainer = "Eelco Dolstra "; - doInstallCheck = true; - #enableParallelBuilding = true; - }; - - -in jobs diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 34abf05ea..000000000 --- a/shell.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ useClang ? true -, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz -}: - -with import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; - -with import ./release-common.nix { inherit pkgs; }; - -(if useClang then clangStdenv else stdenv).mkDerivation { - name = "nix"; - - buildInputs = buildDeps ++ tarballDeps ++ perlDeps; - - inherit configureFlags; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - export prefix=$(pwd)/inst - configureFlags+=" --prefix=$prefix" - PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH - PATH=$prefix/bin:$PATH - unset PYTHONPATH - ''; -} diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index 2345972ce..7211360cd 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -1,6 +1,9 @@ -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; let @@ -101,7 +104,6 @@ makeTest ( { config, pkgs, nodes, ... }: { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ]; - nix.package = nix; nix.binaryCaches = [ ]; environment.systemPackages = [ pkgs.jq ]; networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = @@ -135,11 +137,11 @@ makeTest ( my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified"); strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModified}" or die "time mismatch"; - $client->succeed("nix build nixpkgs:hello"); + $client->succeed("nix build nixpkgs#hello"); # The build shouldn't fail even with --tarball-ttl 0 (the server # being down should not be a fatal error). - $client->succeed("nix build nixpkgs:fuse --tarball-ttl 0"); + $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0"); ''; }) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 0dc147fb3..7c4bdaded 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -1,8 +1,11 @@ # Test ‘nix-copy-closure’. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { @@ -11,7 +14,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { { config, pkgs, ... }: { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgA ]; - nix.package = nix; nix.binaryCaches = [ ]; }; @@ -20,7 +22,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { { services.openssh.enable = true; virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgB pkgC ]; - nix.package = nix; }; }; diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index b867f13b4..a53f9bfcd 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -1,8 +1,11 @@ # Test Nix's remote build feature. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest ( @@ -13,7 +16,6 @@ let { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; - nix.package = nix; nix.useSandbox = true; }; @@ -59,7 +61,6 @@ in ]; virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; - nix.package = nix; nix.binaryCaches = [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; }; diff --git a/tests/setuid.nix b/tests/setuid.nix index 77e83c8d6..d79d071df 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -1,15 +1,17 @@ # Verify that Linux builds cannot create setuid or setgid binaries. -{ nixpkgs, system, nix }: +{ nixpkgs, system, overlay }: -with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; +with import (nixpkgs + "/nixos/lib/testing.nix") { + inherit system; + extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ]; +}; makeTest { machine = { config, lib, pkgs, ... }: { virtualisation.writableStore = true; - nix.package = nix; nix.binaryCaches = [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; From 93b1ce1ac522ff92626da5cdd689008063935844 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 16:34:59 +0200 Subject: [PATCH 0521/2548] Revert "std::uncaught_exception() -> std::uncaught_exceptions()" This reverts commit 6b83174ffffbdfc3f876d94d5178e0b83f675cae because it doesn't work on macOS yet. https://hydra.nixos.org/build/102617587 --- src/libstore/remote-store.cc | 2 +- src/libutil/json.hh | 2 +- src/libutil/util.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b9e7a80ba..b7f202a6b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -229,7 +229,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exceptions()) { + if (!daemonException && std::uncaught_exception()) { handle.markBad(); debug("closing daemon connection because of an exception"); } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 45a22f011..02a39917f 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -170,7 +170,7 @@ public: ~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + assert(!first || std::uncaught_exception()); } template diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e65f8ee56..1b7449991 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1169,7 +1169,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { + if (!interruptThrown && !std::uncaught_exception()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } From 21e2088c1b6f5458f31fd368f416d45e4eb2bc5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 17:25:59 +0200 Subject: [PATCH 0522/2548] nix-perl -> nix.perl-bindings --- flake.nix | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/flake.nix b/flake.nix index 220b6d8c5..0f8da30e2 100644 --- a/flake.nix +++ b/flake.nix @@ -93,11 +93,11 @@ in { - # A Nixpkgs overlay that overrides the 'nix' and 'nix-perl' - # packages. + # A Nixpkgs overlay that overrides the 'nix' and + # 'nix.perl-bindings' packages. overlay = final: prev: { - nix = with final; with commonDeps pkgs; releaseTools.nixBuild { + nix = with final; with commonDeps pkgs; (releaseTools.nixBuild { name = "nix"; src = self.hydraJobs.tarball; @@ -127,24 +127,26 @@ doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; - }; + }) // { - nix-perl = with final; releaseTools.nixBuild { - name = "nix-perl"; - src = self.hydraJobs.tarball; + perl-bindings = with final; releaseTools.nixBuild { + name = "nix-perl"; + src = self.hydraJobs.tarball; - buildInputs = - [ nix curl bzip2 xz pkgconfig pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + buildInputs = + [ nix curl bzip2 xz pkgconfig pkgs.perl boost ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; - configureFlags = '' - --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} - --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} - ''; + configureFlags = '' + --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} + --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} + ''; - enableParallelBuilding = true; + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; + }; - postUnpack = "sourceRoot=$sourceRoot/perl"; }; }; @@ -197,7 +199,7 @@ build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix); # Perl bindings for various platforms. - perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix-perl); + perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -429,7 +431,7 @@ }; packages = { - inherit (nixpkgsFor.x86_64-linux) nix nix-perl; + inherit (nixpkgsFor.x86_64-linux) nix; }; defaultPackage = self.packages.nix; From c5bd564c69ec1fb4f2ffc1091653d99ebc87e284 Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Sun, 6 Oct 2019 16:54:56 -0400 Subject: [PATCH 0523/2548] nix doctor: add more logging output to checks When running nix doctor on a healthy system, it just prints the store URI and nothing else. This makes it unclear whether the system is in a good state and what check(s) it actually ran, since some of the checks are optional depending on the store type. This commit updates nix doctor to print an colored log message for every check that it does, and explicitly state whether that check was a PASS or FAIL to make it clear to the user whether the system passed its checkup with the doctor. Fixes #3084 --- src/nix/doctor.cc | 67 +++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 7b5444619..795236c5f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -1,11 +1,17 @@ +#include + #include "command.hh" +#include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" +#include "util.hh" #include "worker-protocol.hh" using namespace nix; +namespace { + std::string formatProtocol(unsigned int proto) { if (proto) { @@ -16,6 +22,18 @@ std::string formatProtocol(unsigned int proto) return "unknown"; } +bool checkPass(const std::string & msg) { + logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + return true; +} + +bool checkFail(const std::string & msg) { + logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + return false; +} + +} + struct CmdDoctor : StoreCommand { bool success = true; @@ -27,13 +45,12 @@ struct CmdDoctor : StoreCommand std::string description() override { - return "check your system for potential problems"; + return "check your system for potential problems and print a PASS or FAIL for each check."; } void run(ref store) override { - std::cout << "Store uri: " << store->getUri() << std::endl; - std::cout << std::endl; + logger->log("Running checks against store uri: " + store->getUri()); auto type = getStoreType(); @@ -56,15 +73,14 @@ struct CmdDoctor : StoreCommand dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); if (dirs.size() != 1) { - std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; - std::cout << std::endl; + std::stringstream ss; + ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; + ss << " " << dir << "\n"; + return checkFail(ss.str()); } - return true; + return checkPass("PATH contains only one nix version."); } bool checkProfileRoots(ref store) @@ -87,17 +103,17 @@ struct CmdDoctor : StoreCommand } if (!dirs.empty()) { - std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; - std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; - std::cout << "garbage collected, resulting in broken symlinks." << std::endl; - std::cout << std::endl; + std::stringstream ss; + ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n" + << "The generation this profile points to might not have a gcroot and could be\n" + << "garbage collected, resulting in broken symlinks.\n\n"; for (auto & dir : dirs) - std::cout << " " << dir << std::endl; - std::cout << std::endl; - return false; + ss << " " << dir << "\n"; + ss << "\n"; + return checkFail(ss.str()); } - return true; + return checkPass("All profiles are gcroots."); } bool checkStoreProtocol(unsigned int storeProto) @@ -107,17 +123,16 @@ struct CmdDoctor : StoreCommand : PROTOCOL_VERSION; if (clientProto != storeProto) { - std::cout << "Warning: protocol version of this client does not match the store." << std::endl; - std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; - std::cout << "sync with the daemon." << std::endl; - std::cout << std::endl; - std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; - std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; - std::cout << std::endl; - return false; + std::stringstream ss; + ss << "Warning: protocol version of this client does not match the store.\n" + << "While this is not necessarily a problem it's recommended to keep the client in\n" + << "sync with the daemon.\n\n" + << "Client protocol: " << formatProtocol(clientProto) << "\n" + << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; + return checkFail(ss.str()); } - return true; + return checkPass("Client protocol matches store protocol."); } }; From 6f6cb5e3880d0c7a1dd2bc13c2e0be8ce0ae9fa1 Mon Sep 17 00:00:00 2001 From: Sam Doshi Date: Mon, 7 Oct 2019 11:40:42 +0100 Subject: [PATCH 0524/2548] nix search: remove verbose example --- src/nix/search.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/nix/search.cc b/src/nix/search.cc index e086de226..764dcb70a 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -80,10 +80,6 @@ struct CmdSearch : SourceExprCommand, MixJSON Example{ "To search for git and frontend or gui:", "nix search git 'frontend|gui'" - }, - Example{ - "To display the description of the found packages:", - "nix search git --verbose" } }; } From ce2c755d2aa4d970c2de8f00f9ed2099ad991255 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Oct 2019 14:02:52 +0200 Subject: [PATCH 0525/2548] Add a "dev" output to the 'nix' package This is to ensure that references like 'nix.dev' in dwarffs work regardless of whether we're using the 'nix' package from this overlay or from Nixpkgs. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 0f8da30e2..4ec80e9d6 100644 --- a/flake.nix +++ b/flake.nix @@ -101,6 +101,8 @@ name = "nix"; src = self.hydraJobs.tarball; + outputs = [ "out" "dev" ]; + buildInputs = buildDeps; preConfigure = From a15f9b37eba805c2b3bd37844c0a527b14774bba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 7 Oct 2019 15:44:32 +0200 Subject: [PATCH 0526/2548] fetchGit: Support Git trees without any commits Fixes $ nix build fatal: bad revision 'HEAD' error: program 'git' failed with exit code 128 on a new flake. It is now detected as a dirty tree with revCount = 0. --- src/libexpr/primops/fetchGit.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 64f138195..21fa025c1 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -33,13 +33,19 @@ GitInfo exportGit(ref store, std::string uri, // or revision is given, then allow the use of an unclean working // tree. if (!ref && !rev && isLocal) { - bool clean = true; + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + bool haveCommits = !readDirectory(uri + "/.git/refs/heads").empty(); try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); + if (haveCommits) { + runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; } if (!clean) { @@ -75,10 +81,10 @@ GitInfo exportGit(ref store, std::string uri, }; gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })); + gitInfo.revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })) : 0; // FIXME: maybe we should use the timestamp of the last // modified dirty file? - gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" })); + gitInfo.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", uri, "show", "-s", "--format=%ct", "HEAD" })) : 0; return gitInfo; } From d24bfe29a1edd30900f9205b9446252f3f3836af Mon Sep 17 00:00:00 2001 From: Emilio Karakey Date: Mon, 7 Oct 2019 18:28:10 -0500 Subject: [PATCH 0527/2548] deleted comment --- src/nix/shell.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 5e1d53aff..a4488b229 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -30,7 +30,6 @@ BuildEnvironment readEnvironment(const Path & path) std::set exported; auto file = readFile(path); - //auto file = readFile("/tmp/x"); auto pos = file.cbegin(); From 519aa479d7379981a3ffd914d734c33ee9610efd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2019 14:41:41 +0200 Subject: [PATCH 0528/2548] Remove outdated flakes design doc --- doc/flakes/design.md | 482 ------------------------------------------- 1 file changed, 482 deletions(-) delete mode 100644 doc/flakes/design.md diff --git a/doc/flakes/design.md b/doc/flakes/design.md deleted file mode 100644 index 97bd12ce3..000000000 --- a/doc/flakes/design.md +++ /dev/null @@ -1,482 +0,0 @@ -# Nix Flake MVP - -## Goals - -* Standard and easy way for Nix repos to reference other Nix repos as - dependencies - -* Discoverability: Be able to query and update these references to Nix repos - automatically - -* To provide a replacement for `nix-channel`, `NIX_PATH` and Hydra jobset - definitions - -* Reproducibility: Evaluate packages and NixOS configurations hermetic by - default - -Upcoming but not yet implemented: - -* Sophisticated flake versioning, such as the ability to specify version ranges - on dependencies. - -* A way to specify the types of values provided by a flake. For the most part, - flakes can provide arbitrary Nix values, but there will be some standard - attribute names (e.g. `packages` must be a set of installable derivations). - - -## Overview - -* A flake is (usually) a Git repository that contains a file named `flake.nix` - at top-level - -* A flake *provides* an attribute set of values, such as packages, Nixpkgs - overlays, NixOS modules, library functions, Hydra jobs, `nix-shell` - definitions, etc. - -* Flakes can *depend* on other flakes or other repositories which aren't flakes - -* Flakes are referred to using a *flake reference*, which is either a URL - specifying its repository's location or an identifier looked up in a *lock - file* or *flake registry*. - -* A *flake registry* is a mapping from flake identifiers to flake locations - (e.g. `nixpkgs -> github:NixOS/nixpkgs/release-18.09`). There is a centrally - maintained flake registry on `nixos.org`. - -* A flake can contain a *lock file* (`flake.lock`) used when resolving the - dependencies in `flake.nix`. It maps mutable flake references - (e.g. `github:NixOS/nixpkgs/release-18.09`) to references containing revisions - (e.g. `nixpkgs -> - github:NixOS/nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f`). - -* The `nix` command uses the flake registry as its default installation source. - For example, `nix build nixpkgs.hello` builds the `hello` package provided by - the `nixpkgs` flake listed in the registry. `nix` will automatically - download/upload the registry and flakes as needed. - -* `nix build` without arguments will build the flake in the current - directory (or some parent). - -* `nix flake update` generates `flake.lock` from `flake.nix`, ignoring the old - lockfile. - -* `nixos-rebuild` will build a configuration from a (locked) flake. Evaluation - is done in pure mode to ensure there are no unaccounted inputs. Thus the - NixOS configuration can be reproduced unambiguously from the top-level flake. - -* Nix code can query flake metadata such as `commitHash` (the Git revision) or - `edition` (the date of the last commit). This is useful for NixOS to compute - the NixOS version string (which will be the revision of the top-level - configuration flake, uniquely identifying the configuration). - -* Hydra jobset configurations will consist of a single flake reference. Thus we - can get rid of jobset inputs; any other needed repositories can be fetched by - the top-level flake. The top-level flake can be locked or unlocked; if some - dependencies are unlocked, then Nix will fetch the latest revision for each. - - -## Example flake - -Let us look at an example of a `flake.nix` file, here for `dwarffs`, a small -repository that provides a single package and a single NixOS module. - -```nix -{ - # The flake identifier. - name = "dwarffs"; - - # The edition may be used in the future to determine how Nix - # expressions inside this flake are to be parsed. - edition = 201906; - - # Some other metadata. - description = "A filesystem that fetches DWARF debug info from the Internet on demand"; - - # The flake dependencies. Nix will resolve and fetch these flakes and pass - # them as a function argument to `outputs` below. - # - # "nixpkgs" denotes a flake named `nixpkgs` which is looked up - # in the flake registry, or in `flake.lock` inside this flake, if it - # exists. - inputs = [ flake:nixpkgs ]; - - # An attribute set listing dependencies which aren't flakes, also to be passed as - # a function argument to `provides`. - nonFlakeRequires = {}; - - # The stuff provided by this flake. Flakes can provide whatever they - # want (convention over configuration), but some attributes have - # special meaning to tools / other flakes. For example, `packages` - # is used by the `nix` CLI to search for packages, and - # `nixosModules` is used by NixOS to automatically pull in the - # modules provided by a flake. - # - # `outputs` takes a single argument (`deps`) that contains - # the resolved set of flakes. (See below.) - outputs = deps: { - - # This is searched by `nix`, so something like `nix install - # dwarffs.dwarffs` resolves to this `packages.dwarffs`. - packages.dwarffs = - with deps.nixpkgs.packages; - with deps.nixpkgs.builders; - with deps.nixpkgs.lib; - - stdenv.mkDerivation { - name = "dwarffs-0.1"; - - buildInputs = [ fuse nix nlohmann_json boost ]; - - NIX_CFLAGS_COMPILE = "-I ${nix.dev}/include/nix -include ${nix.dev}/include/nix/config.h -D_FILE_OFFSET_BITS=64"; - - src = cleanSource ./.; - - installPhase = - '' - mkdir -p $out/bin $out/lib/systemd/system - - cp dwarffs $out/bin/ - ln -s dwarffs $out/bin/mount.fuse.dwarffs - - cp ${./run-dwarffs.mount} $out/lib/systemd/system/run-dwarffs.mount - cp ${./run-dwarffs.automount} $out/lib/systemd/system/run-dwarffs.automount - ''; - }; - - # NixOS modules. - nixosModules.dwarffs = import ./module.nix deps; - - # Provide a single Hydra job (`hydraJobs.dwarffs`). - hydraJobs.build.x86_64-linux = packages.dwarffs; - - # A bunch of things which can be checked (through `nix flake check`) to - # make sure the flake is well-defined. - checks.build = packages.dwarffs; - }; -} -``` - -Similarly, a minimal `flake.nix` for Nixpkgs: - -```nix -{ - name = "nixpkgs"; - - edition = 201906; - - description = "A collection of packages for the Nix package manager"; - - outputs = deps: - let pkgs = import ./. {}; in - let pkgs = import ./. { system = "x86_64-linux"; }; in - { - lib = (import ./lib) // { - nixosSystem = import ./nixos/lib/eval-config.nix; - }; - - builders = { - inherit (pkgs) stdenv fetchurl; - }; - - packages = { - inherit (pkgs) hello nix fuse nlohmann_json boost; - }; - - legacyPkgs = pkgs; - }; -} -``` -Note that `packages` is an unpolluted set of packages: non-package values like -`lib` or `fetchurl` are not part of it. - -## Flake registries - -Note: If a flake registry contains an entry `nixpkgs -> github:NixOS/nixpkgs`, -then `nixpkgs/release-18.09` will match to become -`github:NixOS/nixpkgs/release-18.09`. This is referred to as "fuzzymatching". - - -## Flake references - -Flake references are a URI-like syntax to specify the physical location of a -flake (e.g. a Git repository) or to denote a lookup in the flake registry or -lock file. There are four options for the syntax: - -* Flake aliases - A flake alias is a name which requires a lookup in a flake - registry or lock file. - - Example: "nixpkgs" - -* GitHub repositories - A repository which is stored on GitHub can easily be fetched using this type. - Note: - * Only the code in this particular commit is downloaded, not the entire repo - * By default, the commit to download is the last commit on the `master` branch. - See later for how to change this. - - Example: `github:NixOS/nixpkgs` - -* `ssh/https/git/file` - These are generic `FlakeRef`s for downloadding git repositories or tarballs. - - Examples: - - https://example.org/my/repo.git - - ssh://git@github.com:NixOS/nix.git - - git://github.com/edolstra/dwarffs.git - - file:///home/my-user/some-repo/some-repo.git - - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz - - file:///.tar.xz - -* Local, dirty paths - This `FlakeRef` is the equivalent of `file://` used for dirty paths. - - Example: /path/to/my/repo - -Notes: -- Each FlakeRef (except for the Path option) allows for a Git revision (i.e. - commit hash) and/or referenceo(i.e. git branch name) to be added. For - tarbals, an SRI hash needs to be added. - Examples: - * `"nixpkgs/release-18.09"` - * `github:NixOS/nixpkgs/1e9e709953e315ab004951248b186ac8e2306451` - * `git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817` - * file:///.tar.xz(?hash=) -- In full pure mode, no mutable `FlakeRef`s can be used - * No aliases, because they need to be looked up - * `github` requires a specified `rev` - * `ssh/https/git/file` require a specified `ref` _and_ `rev` - * `path` is always mutable -- Flakes don't need to be top-level, but can also reside in a subdirectory. This is shown by adding `dir=` to the `FlakeRef`. - Example: `./foo?dir=bar` - - -## Flake lock files - -A lockfile is a JSON file named `flake.lock` which contains a forrest of -entries mapping `FlakeRef`s to the immutable `FlakeRef` they were resolved to. - -Example: - -```json -{ - "nixpkgs": { - "uri": "github:NixOS/nixpkgs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553", - "content-hash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=" - }, - "foo": { - "uri": "https://example.org/foo.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c", - "content-hash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=" - } -} -``` - -Lockfiles are used to help resolve the dependencies of a flake. -- `nix build github:<..>` uses the remote lockfile and update it -- `nix build /home/user/dwarffs` uses the local lockfile, updates it and writes the result to file -- `nix flake update ` recreates the lockfile from scratch and writes it to file -- `--no-registries` makes the command pure, also when fetching dependencies -- `--no-save-lock-file`: Several commands will update the lockfile (e.g. `nix - build`). This flag prevents the updated lockfile to be written to file. -- `--recreate-lock-file` makes prevents the current lockfile from being used - -## `outputs` - -The function argument `deps` is an attrset containing all dependencies listed -in `requires` and `nonFlakeRequires` as well as `path` (for the flake's source -code) and an attribute `meta` with: -- `description` -- `commitHash` (not for tarball flakes): The Git commit hash. -- `date`: The timestamp of the most recent commit (for Git repos), or of the - most recently modified file (for tarballs) -- `revCount` (for Git flakes, but not GitHub flakes): The number of ancestors - of the revision. Useful for generating version strings. - -The flake attribute `outputs` is a function that takes an argument named `deps` -and returns an attribute set. Some of the members of this set have protected -names: - -* `packages`: A set of installable derivations used by the `nix` command. That - is, commands such as `nix install` ignore all other flake attributes. It - cannot be a nested set. - -* `hydraJobs`: Used by Hydra. - -* `nixosModules`: An attrset of NixOS modules. - -* `nixosSystems`: An attrset of calls to `evalModules`, i.e. things - that `nixos-rebuild` can switch to. (Maybe this is superfluous, but - we need to avoid a situation where `nixos-rebuild` needs to fetch - its own `nixpkgs` just to do `evalModules`.) - -* `devShell`: A derivation to create a development environment - -* `self`: The result of the flake's output which is passed to itself - Example: `self.outputs.foo` works. - - -## Flake registry - -A flake registry is a JSON file mapping flake references to flake references. -The default/global registry is kept at -`https://github.com/NixOS/flake-registry/blob/master/flake-registry.json` and -looks like this: - -```json -{ - "flakes": { - "dwarffs": { - "uri": "github:edolstra/dwarffs/flake" - }, - "nix": { - "uri": "github:NixOS/nix/flakes" - }, - "nixpkgs": { - "uri": "github:edolstra/nixpkgs/release-19.03" - }, - "hydra": { - "uri": "github:NixOS/hydra/flake" - }, - "patchelf": { - "uri": "github:NixOS/patchelf" - } - }, - "version": 1 -} -``` - -Nix automatically (re)downloads this file whenever you have network access. The -downloaded file is a GC root so the registry remains available if nixos.org is -unreachable. - -In addition to a global registry, there is also a user registry stored in -`~/.config/nix/registry.json`. - - -## Nix UI - -There is a list of new commands added to the `nix` CLI: - -* `nix flake list`: Show all flakes in the registry - -* `nix flake add `: Add or override a flake - to/in the user flake registry. - -* `nix flake remove `: Remove a FlakeRef from the user flake - registry. - -* `nix flake pin `: Look up to which immutable FlakeRef the - alias FlakeRef maps to currently, and store that map in the user registry. - Example: `nix flake pin github:NixOS/nixpkgs` will create an entry - `github:NixOS/nixpkgs -> - github:NixOS/nixpkgs/444f22ca892a873f76acd88d5d55bdc24ed08757`. - -* `nix flake init`: Create a `flake.nix` in the current directory - -* `nix flake update`: Recreate the lock file from scratch, from the `flake.nix`. - -* `nix flake check`: Do some checks on the flake, e.g. check that all - `packages` are really packages. - -* `nix flake clone`: `git clone` the flake repo - -Flags / configuration options: - -* `--flakes (=)*`: add/override some - FlakeRef - -* `--flake `: set the specified flake as the installation source - E.g. `nix build --flake ./my-nixpkgs hello`. - -The default installation source in `nix` is the `packages` from all flakes in -the registry, that is: -``` -builtins.mapAttrs (flakeName: flakeInfo: - (getFlake flakeInfo.uri).${flakeName}.outputs.packages or {}) - builtins.flakeRegistry -``` -where `builtins.flakeRegistry` is the global registry with user overrides -applied, and `builtins.getFlake` downloads a flake and resolves its -dependencies. - - -## Pure evaluation and caching - -Flake evaluation is done in pure mode. Thus: - -* Flakes cannot use `NIX_PATH` via the `<...>` syntax. - -* Flakes cannot read random stuff from non-flake directories, such as - `~/.nix/config.nix` or overlays. - -This enables aggressive caching or precomputation of Nixpkgs package sets. For -example, for a particular Nixpkgs flake closure (as identified by, say, a hash -of the fully-qualified flake references after dependency resolution) and system -type, an attribute like `packages.hello` should always evaluate to the same -derivation. So we can: - -* Keep a local evaluation cache (say `~/.cache/nix/eval-cache-v1.sqlite`) - mapping `() -> (, - , )`. - -* Download a precomputed cache, e.g. - `https://releases.nixos.org/eval/.sqlite`. So a command - like `nix search` could avoid evaluating Nixpkgs entirely. - -Of course, this doesn't allow overlays. With pure evaluation, the only way to -have these is to define a top-level flake that depends on the Nixpkgs flake and -somehow passes in a set of overlays. - - -## Hydra jobset dependencies - -Hydra can use the flake dependency resolution mechanism to fetch dependencies. -This allows us to get rid of jobset configuration in the web interface: a -jobset only requires a flake reference. That is, a jobset *is* a flake. Hydra -then just builds the `hydraJobs` attrset - - -## NixOS system configuration - -NixOS currently contains a lot of modules that really should be moved into -their own repositories. For example, it contains a Hydra module that duplicates -the one in the Hydra repository. Also, we want reproducible evaluation for -NixOS system configurations. So NixOS system configurations should be stored as -flakes in (local) Git repositories. - -`my-system/flake.nix`: -```nix -{ - name = "my-system"; - - edition = 201906; - - inputs = - [ "nixpkgs/nixos-18.09" - "dwarffs" - "hydra" - ... lots of other module flakes ... - ]; - - outputs = flakes: { - nixosSystems.default = - flakes.nixpkgs.lib.evalModules { - modules = - [ { networking.firewall.enable = true; - hydra.useSubstitutes = true; - } - # The latter could be extracted automatically from `flakes`. - flakes.dwarffs.nixosModules.dwarffs - flakes.hydra.nixosModules.hydra - ]; - }; - }; -} -``` - -We can then build the system: -``` -nixos-rebuild switch --flake ~/my-system -``` -This performs dependency resolution starting at `~/my-system/flake.nix` and -builds the `system` attribute in `nixosSystems.default`. From 21304c11f926892b1a0098ba5d424445d5ae30d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2019 16:30:04 +0200 Subject: [PATCH 0529/2548] uri -> url for consistency --- flake.lock | 4 ++-- flake.nix | 2 +- src/libexpr/flake/flake.cc | 17 ++++++++++++----- src/libexpr/flake/lockfile.cc | 8 ++++---- src/nix/flake.cc | 20 ++++++++++---------- tests/flakes.sh | 22 +++++++++++----------- 6 files changed, 40 insertions(+), 33 deletions(-) diff --git a/flake.lock b/flake.lock index 05e3a6e25..9c11ec4fc 100644 --- a/flake.lock +++ b/flake.lock @@ -3,8 +3,8 @@ "nixpkgs": { "inputs": {}, "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", - "originalUri": "nixpkgs/release-19.03", - "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" + "originalUrl": "nixpkgs/release-19.03", + "url": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" } }, "version": 3 diff --git a/flake.nix b/flake.nix index 4ec80e9d6..167064d72 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.uri = "nixpkgs/release-19.03"; + inputs.nixpkgs.url = "nixpkgs/release-19.03"; outputs = { self, nixpkgs }: diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 9e260263c..e8eb353fb 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -34,8 +34,14 @@ std::shared_ptr readRegistry(const Path & path) throw Error("flake registry '%s' has unsupported version %d", path, version); auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) - registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + // FIXME: remove 'uri' soon. + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.emplace(i.key(), url); + } return registry; } @@ -46,7 +52,7 @@ void writeRegistry(const FlakeRegistry & registry, const Path & path) nlohmann::json json; json["version"] = 1; for (auto elem : registry.entries) - json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; + json["flakes"][elem.first.to_string()] = { {"url", elem.second.to_string()} }; createDirs(dirOf(path)); writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. } @@ -283,7 +289,8 @@ static Flake getFlake(EvalState & state, const FlakeRef & originalRef, } auto sInputs = state.symbols.create("inputs"); - auto sUri = state.symbols.create("uri"); + auto sUrl = state.symbols.create("url"); + auto sUri = state.symbols.create("uri"); // FIXME: remove soon auto sFlake = state.symbols.create("flake"); if (std::optional inputs = vInfo.attrs->get(sInputs)) { @@ -295,7 +302,7 @@ static Flake getFlake(EvalState & state, const FlakeRef & originalRef, FlakeInput input(FlakeRef(inputAttr.name)); for (Attr attr : *(inputAttr.value->attrs)) { - if (attr.name == sUri) { + if (attr.name == sUrl || attr.name == sUri) { expectType(state, tString, *attr.value, *attr.pos); input.ref = std::string(attr.value->string.s); } else if (attr.name == sFlake) { diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 039b7a7c1..2c3f3c93d 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -5,8 +5,8 @@ namespace nix::flake { LockedInput::LockedInput(const nlohmann::json & json) : LockedInputs(json) - , ref(json["uri"]) - , originalRef(json["originalUri"]) + , ref(json.value("url", json.value("uri", ""))) + , originalRef(json.value("originalUrl", json.value("originalUri", ""))) , narHash(Hash((std::string) json["narHash"])) { if (!ref.isImmutable()) @@ -16,8 +16,8 @@ LockedInput::LockedInput(const nlohmann::json & json) nlohmann::json LockedInput::toJson() const { auto json = LockedInputs::toJson(); - json["uri"] = ref.to_string(); - json["originalUri"] = originalRef.to_string(); + json["url"] = ref.to_string(); + json["originalUrl"] = originalRef.to_string(); json["narHash"] = narHash.to_string(SRI); return json; } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d0135143c..cf4fcf722 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -19,21 +19,21 @@ using namespace nix::flake; class FlakeCommand : virtual Args, public EvalCommand, public MixFlakeOptions { - std::string flakeUri = "."; + std::string flakeUrl = "."; public: FlakeCommand() { - expectArg("flake-uri", &flakeUri, true); + expectArg("flake-url", &flakeUrl, true); } FlakeRef getFlakeRef() { - if (flakeUri.find('/') != std::string::npos || flakeUri == ".") - return FlakeRef(flakeUri, true); + if (flakeUrl.find('/') != std::string::npos || flakeUrl == ".") + return FlakeRef(flakeUrl, true); else - return FlakeRef(flakeUri); + return FlakeRef(flakeUrl); } Flake getFlake() @@ -74,7 +74,7 @@ struct CmdFlakeList : EvalCommand static void printSourceInfo(const SourceInfo & sourceInfo) { - std::cout << fmt("URI: %s\n", sourceInfo.resolvedRef.to_string()); + std::cout << fmt("URL: %s\n", sourceInfo.resolvedRef.to_string()); if (sourceInfo.resolvedRef.ref) std::cout << fmt("Branch: %s\n",*sourceInfo.resolvedRef.ref); if (sourceInfo.resolvedRef.rev) @@ -89,7 +89,7 @@ static void printSourceInfo(const SourceInfo & sourceInfo) static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j) { - j["uri"] = sourceInfo.resolvedRef.to_string(); + j["url"] = sourceInfo.resolvedRef.to_string(); if (sourceInfo.resolvedRef.ref) j["branch"] = *sourceInfo.resolvedRef.ref; if (sourceInfo.resolvedRef.rev) @@ -454,7 +454,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON struct CmdFlakeAdd : MixEvalArgs, Command { FlakeUri alias; - FlakeUri uri; + FlakeUri url; std::string description() override { @@ -464,7 +464,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command CmdFlakeAdd() { expectArg("alias", &alias); - expectArg("flake-uri", &uri); + expectArg("flake-url", &url); } void run() override @@ -473,7 +473,7 @@ struct CmdFlakeAdd : MixEvalArgs, Command Path userRegistryPath = getUserRegistryPath(); auto userRegistry = readRegistry(userRegistryPath); userRegistry->entries.erase(aliasRef); - userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(uri)); + userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(url)); writeRegistry(*userRegistry, userRegistryPath); } }; diff --git a/tests/flakes.sh b/tests/flakes.sh index a91a0c37c..c5e19826c 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -89,19 +89,19 @@ cat > $registry < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Tue, 8 Oct 2019 16:44:09 +0200 Subject: [PATCH 0530/2548] Move addRegistrOverrides --- src/nix/build.cc | 2 -- src/nix/flake.cc | 1 - src/nix/installables.cc | 4 +++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index f63150012..4fd1de026 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,8 +52,6 @@ struct CmdBuild : MixDryRun, MixProfile, InstallablesCommand { auto buildables = build(store, dryRun ? DryRun : Build, installables); - auto evalState = std::make_shared(searchPath, store); - evalState->addRegistryOverrides(registryOverrides); if (dryRun) return; if (outLink != "") { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cf4fcf722..e6dc5680f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -129,7 +129,6 @@ struct CmdFlakeDeps : FlakeCommand void run(nix::ref store) override { auto evalState = getEvalState(); - evalState->addRegistryOverrides(registryOverrides); std::queue todo; todo.push(resolveFlake()); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 867133653..5611a84ae 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -56,8 +56,10 @@ SourceExprCommand::SourceExprCommand() ref EvalCommand::getEvalState() { - if (!evalState) + if (!evalState) { evalState = std::make_shared(searchPath, getStore()); + evalState->addRegistryOverrides(registryOverrides); + } return ref(evalState); } From 00a567588e43b51944a9b91c1e890bda27773aed Mon Sep 17 00:00:00 2001 From: Danny Bautista Date: Tue, 8 Oct 2019 14:02:40 -0400 Subject: [PATCH 0531/2548] Fix typos in the Nix Manual. --- doc/manual/command-ref/conf-file.xml | 2 +- doc/manual/command-ref/nix-env.xml | 2 +- doc/manual/command-ref/nix-prefetch-url.xml | 2 +- doc/manual/expressions/builtins.xml | 4 ++-- doc/manual/expressions/simple-building-testing.xml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 320e15339..48dce7c95 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -433,7 +433,7 @@ builtins.fetchurl { keep-env-derivations If false (default), derivations - are not stored in Nix user environments. That is, the derivation + are not stored in Nix user environments. That is, the derivations of any build-time-only dependencies may be garbage-collected. If true, when you add a Nix derivation to diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index c8c01f9a4..d257a5e49 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -659,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4' gcc-3.3.1 are split into two parts: the package name (gcc), and the version (3.3.1). The version part starts after the first -dash not following by a letter. x is considered an +dash not followed by a letter. x is considered an upgrade of y if their package names match, and the version of y is higher that that of x. diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml index 8ef748c74..621ded72e 100644 --- a/doc/manual/command-ref/nix-prefetch-url.xml +++ b/doc/manual/command-ref/nix-prefetch-url.xml @@ -53,7 +53,7 @@ avoided. If hash is specified, then a download is not performed if the Nix store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an -error if signaled if the actual hash of the file does not match the +error is signaled if the actual hash of the file does not match the specified hash. This command prints the hash on standard output. Additionally, diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index e164d1441..465fa1e0b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -289,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else "" Return element n from the list xs. Elements are counted - starting from 0. A fatal error occurs in the index is out of + starting from 0. A fatal error occurs if the index is out of bounds. @@ -1466,7 +1466,7 @@ in foo A set containing { __toString = self: ...; }. An integer. A list, in which case the string representations of its elements are joined with spaces. - A Boolean (false yields "", true yields "1". + A Boolean (false yields "", true yields "1"). null, which yields the empty string. diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index 0348c082b..7326a3e76 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -43,7 +43,7 @@ use nix-build’s switch to give the symlink another name. -Nix has a transactional semantics. Once a build finishes +Nix has transactional semantics. Once a build finishes successfully, Nix makes a note of this in its database: it registers that the path denoted by out is now valid. If you try to build the derivation again, Nix From d343c03edb8d4d2be7aca4f1a377544b3889b8b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Oct 2019 20:05:16 +0200 Subject: [PATCH 0532/2548] Temporary compatibility hack --- flake.lock | 4 ++-- flake.nix | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 9c11ec4fc..05e3a6e25 100644 --- a/flake.lock +++ b/flake.lock @@ -3,8 +3,8 @@ "nixpkgs": { "inputs": {}, "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", - "originalUrl": "nixpkgs/release-19.03", - "url": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" + "originalUri": "nixpkgs/release-19.03", + "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" } }, "version": 3 diff --git a/flake.nix b/flake.nix index 167064d72..4ec80e9d6 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.url = "nixpkgs/release-19.03"; + inputs.nixpkgs.uri = "nixpkgs/release-19.03"; outputs = { self, nixpkgs }: From 0847f2f1b3145a62dc34707ba788275ce6b6fc57 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 21:17:27 -0400 Subject: [PATCH 0533/2548] Copy instead of linking launch agent On Catalina, the /nix filesystem might not be mounted at start time. To avoid this service not starting, we need to keep the launch agent outside of the Nix store. A wait4pid will hold for our /nix dir to be mounted. Fixes #3125. --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- scripts/install-darwin-multi-user.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index b340610e9..39d7757a4 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -17,7 +17,7 @@ /bin/sh -c - /bin/wait4path @bindir@/nix-daemon && @bindir@/nix-daemon + /bin/wait4path @bindir@/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 87c4c2b05..49076bd5c 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -39,7 +39,7 @@ EOF poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon as a LaunchDaemon" \ - ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" + cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist From d4e51aac080e73982e16b43088f9ae404f07136e Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 21:35:09 -0400 Subject: [PATCH 0534/2548] Make preexisting Nix install a warning, not a failure In the multi-user install script, we originally made sure no previous references to Nix existed. This prevented any previous installs from contaminating the new install. However, some users need the ability to repair their existing Nix installation without uninstalling all references to Nix. This change allows users with existing Nix installations to use the installer, while still outputing a warning message on the dangers of this. As a result, the multi-user install script work much more like the single-user install script has worked in the past. This is a requirement for macOS Catalina users now that /Library/LaunchDaemons/org.nixos.nix-daemon.plisg is not managed by the Nix store. If there is ever a change to the .plist, all users will need to rerun this install script to get the new changes. Otherwise, changes to the launch daemon will require manual interventions. --- scripts/install-multi-user.sh | 104 ++-------------------------------- 1 file changed, 5 insertions(+), 99 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index a41309e93..fd0f54184 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -278,73 +278,9 @@ EOF fi if type nix-env 2> /dev/null >&2; then - failure < Date: Tue, 8 Oct 2019 22:41:05 -0400 Subject: [PATCH 0535/2548] =?UTF-8?q?Don=E2=80=99t=20source=20bashrc=20in?= =?UTF-8?q?=20pure=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure mode should not try to source the user’s bashrc file. These may have many impurities that the user does not expect to get into their shell. Fixes #3090 --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 2b36846cd..3e1081c05 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -412,7 +412,7 @@ static void _main(int argc, char * * argv) auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( (keepTmp ? "" : "rm -rf '%1%'; "s) + - "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " From 199e888785bd23073e44e56f6c74b95dc7c10ffa Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Tue, 8 Oct 2019 23:12:31 -0400 Subject: [PATCH 0536/2548] Handle empty sandbox_shell Previously, SANDBOX_SHELL was set to empty when unavailable. This caused issues when actually generating the sandbox. Instead, just set SANDBOX_SHELL when --with-sandbox-shell= is non-empty. Alternative implementation to https://github.com/NixOS/nix/pull/3038. --- src/libstore/local.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 89fc918c3..d690fea28 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -39,9 +39,12 @@ libstore_CXXFLAGS = \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ - -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" +ifneq ($(sandbox_shell),) +libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" +endif + $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/build.cc: From 8c4a5e7ba1990348b9497c0fc4dc236dda3e7986 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Wed, 9 Oct 2019 08:51:49 +0100 Subject: [PATCH 0537/2548] Make nix-daemon.plist less fragile on macOS We're calling `wait4path` on the full, resolved `@bindir@/nix-daemon` path. That means we're hardcoding something like: /bin/wait4path /nix/store/zs9c5xhp3zv9p23qnjxp87nl5injsi1i-nix-2.3/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon That seems unnecessarily fragile. It might be better to wait4path on the path we intend to call. --- misc/launchd/org.nixos.nix-daemon.plist.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 39d7757a4..9f26296a9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -17,7 +17,7 @@ /bin/sh -c - /bin/wait4path @bindir@/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon StandardErrorPath /var/log/nix-daemon.log From 64d88729005841ff3ccfa75134f9aa5e5828da6d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 14:46:44 +0200 Subject: [PATCH 0538/2548] nix-build: Fix compilation --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 3e1081c05..085650e4f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -412,7 +412,7 @@ static void _main(int argc, char * * argv) auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( (keepTmp ? "" : "rm -rf '%1%'; "s) + - (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " From 7c74f075f4a7274ad38c90085cc269a19a977438 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 14:46:58 +0200 Subject: [PATCH 0539/2548] nix search: Don't quietly ignore errors --- src/nix/search.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/search.cc b/src/nix/search.cc index 764dcb70a..eb75493e4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -261,6 +261,7 @@ struct CmdSearch : SourceExprCommand, MixJSON https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ if (!jsonCacheFile) throw Error("error writing to %s", tmpFile); + throw; } if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) From e6e61f0a54dac0174df996e93fcfedcac7769ab4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 15:36:51 +0200 Subject: [PATCH 0540/2548] getSourceExpr(): Handle channels Fixes #1892. Fixes #1865. Fixes #3119. --- src/nix/installables.cc | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0c1ad3ab3..52f9778fc 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -45,25 +45,26 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) std::unordered_set seen; - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); -#if 0 - auto res = state.resolveSearchPathElem(i); - if (!res.first) continue; - if (!pathExists(res.second)) continue; - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), res.second)); -#endif + auto addEntry = [&](const std::string & name) { + if (name == "") return; + if (!seen.insert(name).second) return; Value * v1 = state.allocValue(); mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); - } + }; + + for (auto & i : searchPath) + /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto & j : readDirectory(i.second)) + if (j.name != "manifest.nix" + && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); vSourceExpr->attrs->sort(); } From 99b73fb5071db9fd757c9927fc3fde34e2abac63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 15:51:52 +0200 Subject: [PATCH 0541/2548] OCD performance fix: {find,count}+insert => insert --- src/libexpr/eval.cc | 24 +++++++------------ src/libexpr/get-drvs.cc | 3 +-- src/libexpr/parser.y | 3 +-- src/libexpr/primops.cc | 6 ++--- src/libexpr/value-to-xml.cc | 5 ++-- src/libstore/build.cc | 11 ++++----- src/libstore/builtins/buildenv.cc | 3 +-- src/libstore/local-store.cc | 3 +-- src/libstore/misc.cc | 9 +++---- src/libstore/references.cc | 3 +-- src/libstore/store-api.cc | 3 +-- src/nix-env/nix-env.cc | 7 ++---- src/nix-store/dotgraph.cc | 7 ++---- src/nix-store/nix-store.cc | 3 +-- src/nix/verify.cc | 3 +-- .../resolve-system-dependencies.cc | 4 +--- 16 files changed, 32 insertions(+), 65 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9f4b6b411..f60e8d3ab 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -46,11 +46,10 @@ static void printValue(std::ostream & str, std::set & active, con { checkInterrupt(); - if (active.find(&v) != active.end()) { + if (!active.insert(&v).second) { str << ""; return; } - active.insert(&v); switch (v.type) { case tInt: @@ -1446,8 +1445,7 @@ void EvalState::forceValueDeep(Value & v) std::function recurse; recurse = [&](Value & v) { - if (seen.find(&v) != seen.end()) return; - seen.insert(&v); + if (!seen.insert(&v).second) return; forceValue(v); @@ -1865,8 +1863,7 @@ size_t valueSize(Value & v) std::set seen; auto doString = [&](const char * s) -> size_t { - if (seen.find(s) != seen.end()) return 0; - seen.insert(s); + if (!seen.insert(s).second) return 0; return strlen(s) + 1; }; @@ -1874,8 +1871,7 @@ size_t valueSize(Value & v) std::function doEnv; doValue = [&](Value & v) -> size_t { - if (seen.find(&v) != seen.end()) return 0; - seen.insert(&v); + if (!seen.insert(&v).second) return 0; size_t sz = sizeof(Value); @@ -1890,8 +1886,7 @@ size_t valueSize(Value & v) sz += doString(v.path); break; case tAttrs: - if (seen.find(v.attrs) == seen.end()) { - seen.insert(v.attrs); + if (seen.insert(v.attrs).second) { sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); for (auto & i : *v.attrs) sz += doValue(*i.value); @@ -1900,8 +1895,7 @@ size_t valueSize(Value & v) case tList1: case tList2: case tListN: - if (seen.find(v.listElems()) == seen.end()) { - seen.insert(v.listElems()); + if (seen.insert(v.listElems()).second) { sz += v.listSize() * sizeof(Value *); for (size_t n = 0; n < v.listSize(); ++n) sz += doValue(*v.listElems()[n]); @@ -1922,8 +1916,7 @@ size_t valueSize(Value & v) sz += doValue(*v.primOpApp.right); break; case tExternal: - if (seen.find(v.external) != seen.end()) break; - seen.insert(v.external); + if (!seen.insert(v.external).second) break; sz += v.external->valueSize(seen); break; default: @@ -1934,8 +1927,7 @@ size_t valueSize(Value & v) }; doEnv = [&](Env & env) -> size_t { - if (seen.find(&env) != seen.end()) return 0; - seen.insert(&env); + if (!seen.insert(&env).second) return 0; size_t sz = sizeof(Env) + sizeof(Value *) * env.size; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 21a4d7917..4e22a1d77 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -277,8 +277,7 @@ static bool getDerivation(EvalState & state, Value & v, /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); + if (!done.insert(v.attrs).second) return false; DrvInfo drv(state, attrPath, v.attrs); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 967c88d9b..1dcd532ad 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -138,11 +138,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { - if (formals->argNames.find(formal.name) != formals->argNames.end()) + if (!formals->argNames.insert(formal.name).second) throw ParseError(format("duplicate formal function argument '%1%' at %2%") % formal.name % pos); formals->formals.push_front(formal); - formals->argNames.insert(formal.name); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d4c60f870..deaffa605 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -396,8 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar throw EvalError(format("attribute 'key' required, at %1%") % pos); state.forceValue(*key->value); - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); + if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ @@ -1273,13 +1272,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { + if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 00b1918a8..1f0b1541d 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "derivation", xmlAttrs); - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); + if (drvPath != "" && drvsSeen.insert(drvPath).second) showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else + else doc.writeEmptyElement("repeated"); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9a6729f9e..0e0f8a545 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -568,10 +568,9 @@ UserLock::UserLock() { auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(fnUserLock)) + if (!lockedPaths->insert(fnUserLock).second) /* We already have a lock on this one. */ continue; - lockedPaths->insert(fnUserLock); } try { @@ -620,8 +619,8 @@ UserLock::UserLock() UserLock::~UserLock() { auto lockedPaths(lockedPaths_.lock()); - assert(lockedPaths->count(fnUserLock)); - lockedPaths->erase(fnUserLock); + auto erased = lockedPaths->erase(fnUserLock); + assert(erased); } @@ -1125,10 +1124,8 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) needRestart = true; } else for (auto & i : outputs) - if (wantedOutputs.find(i) == wantedOutputs.end()) { - wantedOutputs.insert(i); + if (wantedOutputs.insert(i).second) needRestart = true; - } } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 74e706664..096593886 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -123,8 +123,7 @@ static Path out; static void addPkg(const Path & pkgDir, int priority) { - if (done.count(pkgDir)) return; - done.insert(pkgDir); + if (!done.insert(pkgDir).second) return; createLinks(pkgDir, out, priority); try { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a2af51d0e..859094e61 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1292,8 +1292,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, { checkInterrupt(); - if (done.find(path) != done.end()) return; - done.insert(path); + if (!done.insert(path).second) return; if (!isStorePath(path)) { printError(format("path '%1%' is not in the Nix store") % path); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index dddf13430..05b93d4c9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -29,8 +29,7 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (state->paths.count(path)) return; - state->paths.insert(path); + if (!state->paths.insert(path).second) return; state->pending++; } @@ -175,8 +174,7 @@ void Store::queryMissing(const PathSet & targets, { auto state(state_.lock()); - if (state->done.count(path)) return; - state->done.insert(path); + if (!state->done.insert(path).second) return; } DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); @@ -252,8 +250,7 @@ Paths Store::topoSortPaths(const PathSet & paths) if (parents.find(path) != parents.end()) throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); - if (visited.find(path) != visited.end()) return; - visited.insert(path); + if (!visited.insert(path).second) return; parents.insert(path); PathSet references; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 5b7eb1f84..0dcc264c3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -36,11 +36,10 @@ static void search(const unsigned char * s, size_t len, } if (!match) continue; string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { + if (hashes.erase(ref)) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i); seen.insert(ref); - hashes.erase(ref); } ++i; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5f63c53b5..be13fa49a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -954,8 +954,7 @@ std::list> getDefaultSubstituters() StringSet done; auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); + if (!done.insert(uri).second) return; try { stores.push_back(openStore(uri)); } catch (Error & e) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 87b2e43f0..48686ce72 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -124,11 +124,10 @@ static void getAllExprs(EvalState & state, string attrName = i; if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { + if (!attrs.insert(attrName).second) { printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); continue; } - attrs.insert(attrName); /* Load the expression on demand. */ Value & vFun = state.getBuiltin("import"); Value & vArg(*state.allocValue()); @@ -307,10 +306,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, /* Insert only those elements in the final list that we haven't inserted before. */ for (auto & j : matches) - if (done.find(j.second) == done.end()) { - done.insert(j.second); + if (done.insert(j.second).second) elems.push_back(j.first); - } } checkSelectorUse(selectors); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index abdfa5e58..d448654fe 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -71,9 +71,7 @@ void printClosure(const Path & nePath, const StoreExpr & fs) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - + if (doneSet.insert(path).second) { ClosureElems::const_iterator elem = fs.closure.elems.find(path); if (elem == fs.closure.elems.end()) throw Error(format("bad closure, missing path '%1%'") % path); @@ -104,8 +102,7 @@ void printDotGraph(ref store, const PathSet & roots) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + if (!doneSet.insert(path).second) continue; cout << makeNode(path, symbolicName(path), "#ff0000"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 0cbceb02f..19384499d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -242,11 +242,10 @@ const string treeNull = " "; static void printTree(const Path & path, const string & firstPad, const string & tailPad, PathSet & done) { - if (done.find(path) != done.end()) { + if (!done.insert(path).second) { cout << format("%1%%2% [...]\n") % firstPad % path; return; } - done.insert(path); cout << format("%1%%2%\n") % firstPad % path; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 7ef571561..2e0d5cffe 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -118,8 +118,7 @@ struct CmdVerify : StorePathsCommand auto doSigs = [&](StringSet sigs) { for (auto sig : sigs) { - if (sigsSeen.count(sig)) continue; - sigsSeen.insert(sig); + if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index e02cfc03e..45e8b6796 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -117,9 +117,7 @@ Path resolveSymlink(const Path & path) std::set resolveTree(const Path & path, PathSet & deps) { std::set results; - if (deps.count(path)) - return {}; - deps.insert(path); + if (!deps.insert(path).second) return {}; for (auto & lib : runResolver(path)) { results.insert(lib); for (auto & p : resolveTree(lib, deps)) { From 926d3e5bb0ca696ab59352b25d0a818707a28186 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 22:57:37 +0200 Subject: [PATCH 0542/2548] Fix Bison 2.4 warning --- src/libexpr/parser.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1dcd532ad..93834bec6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,5 +1,5 @@ %glr-parser -%pure-parser +%define api.pure %locations %define parse.error verbose %defines From 55bba8e4f51758e6c0556b24743e6946c96f6d92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:04:11 +0200 Subject: [PATCH 0543/2548] Make std::uncaught_exception warning less noisy --- src/libutil/json.cc | 5 +++++ src/libutil/json.hh | 5 +---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 0a6fb65f0..74e37b4c4 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -171,4 +171,9 @@ JSONObject JSONPlaceholder::object() return JSONObject(state); } +JSONPlaceholder::~JSONPlaceholder() +{ + assert(!first || std::uncaught_exception()); +} + } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..83213ca66 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -168,10 +168,7 @@ public: { } - ~JSONPlaceholder() - { - assert(!first || std::uncaught_exception()); - } + ~JSONPlaceholder(); template void write(const T & v) From 4331eeb13d241dfe2d2e6a01c53915c556cac94f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:25:06 +0200 Subject: [PATCH 0544/2548] Filter ANSI escape sequences in -L output Otherwise, builds like NixOS VM tests may leave the terminal in a weird state and do resets. --- src/nix/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index c0bcfb0c9..5c05d6b22 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -120,7 +120,7 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { if (state.active) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { auto s2 = s + ANSI_NORMAL "\n"; From 5a303093dcae1e5ce9212616ef18f2ca51020b0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:01:21 +0200 Subject: [PATCH 0545/2548] Remove world-writability from per-user directories 'nix-daemon' now creates subdirectories for users when they first connect. Fixes #509 (CVE-2019-17365). Should also fix #3127. --- nix.spec.in | 2 +- scripts/install-multi-user.sh | 9 ++++----- scripts/nix-profile-daemon.sh.in | 13 ------------- scripts/nix-profile.sh.in | 14 -------------- src/libstore/local-store.cc | 24 ++++++++++++++++++++---- src/libstore/local-store.hh | 2 ++ src/libstore/store-api.hh | 3 +++ src/nix-daemon/nix-daemon.cc | 9 ++++++--- tests/nix-profile.sh | 2 -- tests/remote-store.sh | 4 ++++ tests/user-envs.sh | 2 +- 11 files changed, 41 insertions(+), 43 deletions(-) diff --git a/nix.spec.in b/nix.spec.in index 477768c6a..6b9e37637 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -106,7 +106,7 @@ chmod 1775 $RPM_BUILD_ROOT/nix/store for d in profiles gcroots; do mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user - chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user + chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user done # fix permission of nix profile diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index fd0f54184..258469a0b 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -437,16 +437,15 @@ create_build_users() { } create_directories() { + # FIXME: remove all of this because it duplicates LocalStore::LocalStore(). + _sudo "to make the basic directory structure of Nix (part 1)" \ - mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} + mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user _sudo "to make the basic directory structure of Nix (part 2)" \ - mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user - - _sudo "to make the basic directory structure of Nix (part 3)" \ mkdir -pv -m 1775 /nix/store - _sudo "to make the basic directory structure of Nix (part 4)" \ + _sudo "to make the basic directory structure of Nix (part 3)" \ chgrp "$NIX_BUILD_GROUP_NAME" /nix/store _sudo "to set up the root user's profile (part 1)" \ diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 23da5e855..3e138ac42 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,12 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" -# Set up the per-user profile. -mkdir -m 0755 -p $NIX_USER_PROFILE_DIR -if ! test -O "$NIX_USER_PROFILE_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2 -fi - if test -w $HOME; then if ! test -L $HOME/.nix-profile; then if test "$USER" != root; then @@ -26,13 +20,6 @@ if test -w $HOME; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi - # Create the per-user garbage collector roots directory. - NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER - mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR - if ! test -O "$NIX_USER_GCROOTS_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2 - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 85f1d6e5d..7f9b5877a 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -9,12 +9,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" - - if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 - fi - if [ -w "$HOME" ]; then if ! [ -L "$NIX_LINK" ]; then echo "Nix: creating $NIX_LINK" >&2 @@ -33,14 +27,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" fi - # Create the per-user garbage collector roots directory. - __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" - mkdir -m 0755 -p "$__user_gcroots" - if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 - fi - unset __user_gcroots - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 859094e61..685d43c9c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -74,10 +74,11 @@ LocalStore::LocalStore(const Params & params) multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - Path perUserDir = profilesDir + "/per-user"; - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 01777) == -1) - throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir); + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } mode_t perm = 01775; @@ -1432,4 +1433,19 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } +void LocalStore::createUser(const std::string & userName, uid_t userId) +{ + for (auto & dir : { + fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName) + }) { + createDirs(dir); + if (chmod(dir.c_str(), 0700) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, -1) == -1) + throw SysError("changing owner of directory '%s'", dir); + } +} + + } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3ae34c403..379a06af8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -293,6 +293,8 @@ private: Path getRealStoreDir() override { return realStoreDir; } + void createUser(const std::string & userName, uid_t userId) override; + friend class DerivationGoal; friend class SubstitutionGoal; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7fb568602..ba8990755 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -628,6 +628,9 @@ public: return storePath; } + virtual void createUser(const std::string & userName, uid_t userId) + { } + protected: Stats stats; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index e88aaf636..cd18489b0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -742,7 +742,8 @@ static void performOp(TunnelLogger * logger, ref store, } -static void processConnection(bool trusted) +static void processConnection(bool trusted, + const std::string & userName, uid_t userId) { MonitorFdHup monitor(from.fd); @@ -793,6 +794,8 @@ static void processConnection(bool trusted) params["path-info-cache-size"] = "0"; auto store = openStore(settings.storeUri, params); + store->createUser(userName, userId); + tunnelLogger->stopWork(); to.flush(); @@ -1053,7 +1056,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ from.fd = remote.get(); to.fd = remote.get(); - processConnection(trusted); + processConnection(trusted, user, peer.uid); exit(0); }, options); @@ -1133,7 +1136,7 @@ static int _main(int argc, char * * argv) } } } else { - processConnection(true); + processConnection(true, "root", 0); } } else { daemonLoop(argv); diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b244815e2..5b17fe3fa 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -10,5 +10,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency [ -L $TEST_HOME/.nix-profile ] [ -e $TEST_HOME/.nix-channels ] -[ -e $TEST_ROOT/profile-var/nix/gcroots/per-user/$user ] -[ -e $TEST_ROOT/profile-var/nix/profiles/per-user/$user ] diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f2f2806d0..77437658e 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -13,3 +13,7 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2 nix-store --gc --max-freed 1K killDaemon + +user=$(whoami) +[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] +[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/tests/user-envs.sh b/tests/user-envs.sh index ba6392311..aebf6a2a2 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -20,7 +20,7 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1 # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly -rm -f $HOME/.nix-defexpr +rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr nix-env -qa '*' --description | grep -q silly From c43d9f6131102f2761f22b1ec26f345d357f169c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:05:42 +0200 Subject: [PATCH 0546/2548] Remove some redundant initialization --- scripts/install-multi-user.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 258469a0b..40013eaaf 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -448,12 +448,6 @@ create_directories() { _sudo "to make the basic directory structure of Nix (part 3)" \ chgrp "$NIX_BUILD_GROUP_NAME" /nix/store - _sudo "to set up the root user's profile (part 1)" \ - mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root - - _sudo "to set up the root user's profile (part 2)" \ - mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr" - _sudo "to place the default nix daemon configuration (part 1)" \ mkdir -pv -m 0555 /etc/nix } From 26762ceb8629af95300c0cc8c372a99282060dc1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 18:30:31 +0200 Subject: [PATCH 0547/2548] nix-profile.sh: Don't create .nix-channels This is already done by the installer, so no need to do it again. --- scripts/nix-profile-daemon.sh.in | 5 ----- scripts/nix-profile.sh.in | 5 ----- tests/nix-profile.sh | 1 - 3 files changed, 11 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 3e138ac42..2133a6d3b 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -15,11 +15,6 @@ if test -w $HOME; then fi fi - # Subscribe the root user to the NixOS channel by default. - if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 7f9b5877a..8e70c68b2 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -22,11 +22,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then fi fi - # Subscribe the user to the unstable Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" - fi - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 5b17fe3fa..b808cf1b4 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -9,4 +9,3 @@ USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency [ -L $TEST_HOME/.nix-profile ] -[ -e $TEST_HOME/.nix-channels ] From 9348f9291e5d9e4ba3c4347ea1b235640f54fd79 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:21:07 +0200 Subject: [PATCH 0548/2548] nix-env: Create ~/.nix-profile automatically --- scripts/nix-profile-daemon.sh.in | 9 --------- scripts/nix-profile.sh.in | 12 ------------ src/libstore/local-store.cc | 15 ++++++++------- src/libutil/util.cc | 10 ++++++++++ src/libutil/util.hh | 2 ++ src/nix-channel/nix-channel.cc | 8 +------- src/nix-env/nix-env.cc | 11 ++++++++--- tests/nix-channel.sh | 4 ++-- tests/nix-profile.sh | 2 -- 9 files changed, 31 insertions(+), 42 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 2133a6d3b..aa78e6fcc 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -6,15 +6,6 @@ export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" if test -w $HOME; then - if ! test -L $HOME/.nix-profile; then - if test "$USER" != root; then - ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile - fi - fi - # Set up a default Nix expression from which to install stuff. if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then rm -f $HOME/.nix-defexpr diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 8e70c68b2..087f6ee57 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -10,18 +10,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER if [ -w "$HOME" ]; then - if ! [ -L "$NIX_LINK" ]; then - echo "Nix: creating $NIX_LINK" >&2 - if [ "$USER" != root ]; then - if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then - echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 - fi - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" - fi - fi - # Set up a default Nix expression from which to install stuff. __nix_defexpr="$HOME"/.nix-defexpr [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 685d43c9c..2bc04da46 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -70,16 +70,17 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - - for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); - } - mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1b7449991..6f3bf7ae8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -475,6 +475,16 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + static Lazy getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 07c3d28ff..f057fdb2c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -126,6 +126,8 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +std::string getUserName(); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 06eb3d23b..70aa5c966 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -159,13 +159,7 @@ static int _main(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - ; - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER", ""); - if (name.empty()) - throw Error("cannot figure out user name"); - profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; - createDirs(dirOf(profile)); + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); enum { cNone, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 48686ce72..1b412f0bd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1422,9 +1422,14 @@ static int _main(int argc, char * * argv) if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; - globals.profile = pathExists(profileLink) - ? absPath(readLink(profileLink), dirOf(profileLink)) - : canonPath(settings.nixStateDir + "/profiles/default"); + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); } op(globals, opFlags, opArgs); diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 55f1695c4..93f837bef 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -36,7 +36,7 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles rm -f $TEST_HOME/.nix-channels @@ -55,5 +55,5 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b808cf1b4..e2e0d1090 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -7,5 +7,3 @@ rm -rf $TEST_HOME $TEST_ROOT/profile-var mkdir -p $TEST_HOME USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency - -[ -L $TEST_HOME/.nix-profile ] From 61a6176acaa8522cbcf091a34a663ef45307fef7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:38:01 +0200 Subject: [PATCH 0549/2548] nix-profile.sh: Remove coreutils dependency --- scripts/nix-profile.sh.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 087f6ee57..ed4d1a639 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,6 +1,4 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then - __savedpath="$PATH" - export PATH=@coreutils@ # Set up the per-user profile. # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix @@ -47,6 +45,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi - export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK NIX_USER_PROFILE_DIR fi From c9159f86cc9a2fc07e2ab1217c2d8a8824123df4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:50:46 +0200 Subject: [PATCH 0550/2548] nix-env: Create ~/.nix-defexpr automatically --- scripts/nix-profile-daemon.sh.in | 12 ------------ scripts/nix-profile.sh.in | 11 ----------- src/nix-env/nix-env.cc | 19 ++++++++++++------- 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index aa78e6fcc..47655080a 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,18 +5,6 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" -if test -w $HOME; then - # Set up a default Nix expression from which to install stuff. - if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then - rm -f $HOME/.nix-defexpr - mkdir -p $HOME/.nix-defexpr - if [ "$USER" != root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root - fi - fi -fi - - # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index ed4d1a639..e15f7cd46 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -7,17 +7,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - if [ -w "$HOME" ]; then - # Set up a default Nix expression from which to install stuff. - __nix_defexpr="$HOME"/.nix-defexpr - [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" - mkdir -m 0755 -p "$__nix_defexpr" - if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root - fi - unset __nix_defexpr - fi - # Append ~/.nix-defexpr/channels to $NIX_PATH so that # paths work when the user has fetched the Nixpkgs channel. export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1b412f0bd..2aeb8ebb0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -192,12 +192,6 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static Path getDefNixExprPath() -{ - return getHome() + "/.nix-defexpr"; -} - - static long getPriority(EvalState & state, DrvInfo & drv) { return drv.queryMetaInt("priority", 0); @@ -1327,9 +1321,20 @@ static int _main(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getDefNixExprPath(); + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; + if (!pathExists(globals.instSource.nixExprPath)) { + createDirs(globals.instSource.nixExprPath); + replaceSymlink( + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } + globals.dryRun = false; globals.preserveInstalled = false; globals.removeAll = false; From d7bae5680fc26303acb9a9ee1a202f537841a624 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 19:55:05 +0200 Subject: [PATCH 0551/2548] Go back to 755 permission on per-user directories 700 is pointless since the store is world-readable anyway. And per-user/root/channels must be world-readable. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2bc04da46..fbf5dc35c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1441,7 +1441,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) fmt("%s/gcroots/per-user/%s", stateDir, userName) }) { createDirs(dir); - if (chmod(dir.c_str(), 0700) == -1) + if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); if (chown(dir.c_str(), userId, -1) == -1) throw SysError("changing owner of directory '%s'", dir); From 9277e72cb0aac72100c01334fdf25ea79d19052e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 21:04:21 +0200 Subject: [PATCH 0552/2548] Typo --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 40013eaaf..d060e5165 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -490,7 +490,7 @@ EOF We will: - make sure your computer doesn't already have Nix files - (if it does, I will tell you how to clean them up.) + (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users we'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - install Nix in to $NIX_ROOT From 20eec802ff11dd2b152715cd5c81b756d318219d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Oct 2019 23:33:28 +0200 Subject: [PATCH 0553/2548] Force per-user group to a known value --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fbf5dc35c..00f3a31a2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) createDirs(dir); if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, -1) == -1) + if (chown(dir.c_str(), userId, 0) == -1) throw SysError("changing owner of directory '%s'", dir); } } From bda64a2b0f79346012332ed18f5a76388e6d9bae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 00:12:30 +0200 Subject: [PATCH 0554/2548] Doh https://hydra.nixos.org/build/102803044 --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 00f3a31a2..d86f7b1d2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1443,7 +1443,7 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) createDirs(dir); if (chmod(dir.c_str(), 0755) == -1) throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, 0) == -1) + if (chown(dir.c_str(), userId, getgid()) == -1) throw SysError("changing owner of directory '%s'", dir); } } From c3aaf3b8da1a925c569389f13a861816a781a3c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 09:14:05 +0200 Subject: [PATCH 0555/2548] nix-env: Ignore failures creating ~/.nix-profile and ~/.nix-defexpr https://hydra.nixos.org/build/102803093 --- src/nix-env/nix-env.cc | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 2aeb8ebb0..199dc92aa 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1325,14 +1325,16 @@ static int _main(int argc, char * * argv) globals.instSource.systemFilter = "*"; if (!pathExists(globals.instSource.nixExprPath)) { - createDirs(globals.instSource.nixExprPath); - replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), - globals.instSource.nixExprPath + "/channels"); - if (getuid() != 0) + try { + createDirs(globals.instSource.nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), - globals.instSource.nixExprPath + "/channels_root"); + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error &) { } } globals.dryRun = false; @@ -1427,14 +1429,18 @@ static int _main(int argc, char * * argv) if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + globals.profile = profileLink; } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); } op(globals, opFlags, opArgs); From 95cf23ee7c5b0fd69b21811989a5668f4261fd51 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 15:03:01 +0200 Subject: [PATCH 0556/2548] nix verify: Fix uninitialized variable --- src/nix/verify.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2e0d5cffe..74d9673b6 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -13,7 +13,7 @@ struct CmdVerify : StorePathsCommand bool noContents = false; bool noTrust = false; Strings substituterUris; - size_t sigsNeeded; + size_t sigsNeeded = 0; CmdVerify() { @@ -113,7 +113,7 @@ struct CmdVerify : StorePathsCommand else { StringSet sigsSeen; - size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { From 90df25ef7eb5157868a744906eaf6401430bb93d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Oct 2019 15:07:50 +0200 Subject: [PATCH 0557/2548] Fix build --- src/libutil/util.hh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9f8c7092d..814028442 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -122,10 +122,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - std::string getUserName(); /* Return $HOME or the user's home directory from /etc/passwd. */ From 906d56a96b442d4dd8f924c1ce0d1eec0e214af3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Oct 2019 18:48:59 +0200 Subject: [PATCH 0558/2548] ssh-ng: Don't set CPU affinity on the remote Fixes #3138. --- src/libstore/remote-store.cc | 2 +- src/libstore/remote-store.hh | 5 +++++ src/libstore/ssh-store.cc | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7f202a6b..f34369d8f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -151,7 +151,7 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) conn.to << 1 << cpu; else diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 82fbec092..1f375dd71 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: const Setting maxConnectionAge{(Store*) this, std::numeric_limits::max(), "max-connection-age", "number of seconds to reuse a connection"}; + virtual bool sameMachine() = 0; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -146,6 +148,9 @@ public: std::string getUri() override; + bool sameMachine() + { return true; } + private: ref openConnection() override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 39205ae2c..93e117389 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -35,6 +35,9 @@ public: return uriScheme + host; } + bool sameMachine() + { return false; } + void narFromPath(const Path & path, Sink & sink) override; ref getFSAccessor() override; From 499b0388759db0f9f385da402a4bba551268aa99 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sat, 12 Oct 2019 19:02:57 -0400 Subject: [PATCH 0559/2548] Fix sandbox fallback settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tmpDirInSandbox is different when in sandboxed vs. non-sandboxed. Since we don’t know ahead of time here whether sandboxing is enabled, we need to reset all of the env vars we’ve set previously. This fixes the issue encountered in https://github.com/NixOS/nixpkgs/issues/70856. --- src/libstore/build.cc | 60 ++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e0f8a545..273f25363 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -956,6 +956,9 @@ private: /* Fill in the environment for the builder. */ void initEnv(); + /* Setup tmp dir location. */ + void initTmpDir(); + /* Write a JSON file containing the derivation attributes. */ void writeStructuredAttrs(); @@ -2383,6 +2386,7 @@ void DerivationGoal::startBuilder() if (res != 0 && settings.sandboxFallback) { useChroot = false; tmpDirInSandbox = tmpDir; + initTmpDir(); goto fallback; } else if (res != 0) throw Error("unable to start build process"); @@ -2438,32 +2442,7 @@ void DerivationGoal::startBuilder() } -void DerivationGoal::initEnv() -{ - env.clear(); - - /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when - PATH is not set. We don't want this, so we fill it in with some dummy - value. */ - env["PATH"] = "/path-not-set"; - - /* Set HOME to a non-existing path to prevent certain programs from using - /etc/passwd (or NIS, or whatever) to locate the home directory (for - example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd - if HOME is not set, but they will just assume that the settings file - they are looking for does not exist if HOME is set but points to some - non-existing path. */ - env["HOME"] = homeDir; - - /* Tell the builder where the Nix store is. Usually they - shouldn't care, but this is useful for purity checking (e.g., - the compiler or linker might only want to accept paths to files - in the store or in the build directory). */ - env["NIX_STORE"] = worker.store.storeDir; - - /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); - +void DerivationGoal::initTmpDir() { /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing @@ -2501,6 +2480,35 @@ void DerivationGoal::initEnv() inode of the current directory doesn't appear in .. (because getdents returns the inode of the mount point). */ env["PWD"] = tmpDirInSandbox; +} + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); /* Compatibility hack with Nix <= 0.7: if this is a fixed-output derivation, tell the builder, so that for instance `fetchurl' From 96c84937c49435525d0733437aa88902b2c8caf6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 13 Oct 2019 16:41:49 -0400 Subject: [PATCH 0560/2548] Move tmpDirInSandbox to initTmpDir --- src/libstore/build.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 273f25363..03a069ae2 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1961,13 +1961,6 @@ void DerivationGoal::startBuilder() auto drvName = storePathToName(drvPath); tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); - /* In a sandbox, for determinism, always use the same temporary - directory. */ -#if __linux__ - tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; -#else - tmpDirInSandbox = tmpDir; -#endif chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ @@ -2385,7 +2378,6 @@ void DerivationGoal::startBuilder() int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { useChroot = false; - tmpDirInSandbox = tmpDir; initTmpDir(); goto fallback; } else if (res != 0) @@ -2443,6 +2435,14 @@ void DerivationGoal::startBuilder() void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + /* In non-structured mode, add all bindings specified in the derivation via the environment, except those listed in the passAsFile attribute. Those are passed as file names pointing From 0bc8f1669d542ef65fbfa80ea3728f4dd36d63f2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Oct 2019 14:40:16 +0200 Subject: [PATCH 0561/2548] Move code around --- src/nix/command.hh | 49 +------- src/nix/installables.cc | 259 ++++++++++++++++++---------------------- src/nix/installables.hh | 97 +++++++++++++++ 3 files changed, 216 insertions(+), 189 deletions(-) create mode 100644 src/nix/installables.hh diff --git a/src/nix/command.hh b/src/nix/command.hh index 92f606bbe..802dd9828 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,15 +1,15 @@ #pragma once +#include "installables.hh" #include "args.hh" #include "common-eval-args.hh" + #include namespace nix { extern std::string programPath; -struct Value; -class Bindings; class EvalState; class Store; @@ -30,51 +30,6 @@ private: std::shared_ptr _store; }; -struct Buildable -{ - Path drvPath; // may be empty - std::map outputs; -}; - -typedef std::vector Buildables; - -struct App -{ - PathSet context; - Path program; - // FIXME: add args, sandbox settings, metadata, ... - - App(EvalState & state, Value & vApp); -}; - -struct Installable -{ - virtual ~Installable() { } - - virtual std::string what() = 0; - - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } - - Buildable toBuildable(); - - App toApp(EvalState & state); - - virtual Value * toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ - virtual std::optional getStorePath() - { - return {}; - } -}; - struct EvalCommand : virtual StoreCommand, MixEvalArgs { ref getEvalState(); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 5611a84ae..93509955a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,3 +1,4 @@ +#include "installables.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -111,65 +112,58 @@ struct InstallableStorePath : Installable } }; -struct InstallableValue : Installable +std::vector InstallableValue::toDerivations() { - SourceExprCommand & cmd; + auto state = cmd.getEvalState(); - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + auto v = toValue(*state); - virtual std::vector toDerivations() - { - auto state = cmd.getEvalState(); + Bindings & autoArgs = *cmd.getAutoArgs(*state); - auto v = toValue(*state); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + std::vector res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + drvInfo.queryDrvPath(), + drvInfo.queryOutPath(), + drvInfo.queryOutputName() + }); + } - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); + return res; +} - std::vector res; - for (auto & drvInfo : drvInfos) { - res.push_back({ - drvInfo.queryDrvPath(), - drvInfo.queryOutPath(), - drvInfo.queryOutputName() - }); - } +Buildables InstallableValue::toBuildables() +{ + Buildables res; + PathSet drvPaths; + + for (auto & drv : toDerivations()) { + Buildable b{drv.drvPath}; + drvPaths.insert(b.drvPath); + + auto outputName = drv.outputName; + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); + + b.outputs.emplace(outputName, drv.outPath); + + res.push_back(std::move(b)); + } + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{*drvPaths.begin()}; + for (auto & b2 : res) + b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); + return {b}; + } else return res; - } - - Buildables toBuildables() override - { - Buildables res; - - PathSet drvPaths; - - for (auto & drv : toDerivations()) { - Buildable b{drv.drvPath}; - drvPaths.insert(b.drvPath); - - auto outputName = drv.outputName; - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); - - b.outputs.emplace(outputName, drv.outPath); - - res.push_back(std::move(b)); - } - - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{*drvPaths.begin()}; - for (auto & b2 : res) - b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); - return {b}; - } else - return res; - } -}; +} struct InstallableExpr : InstallableValue { @@ -254,123 +248,104 @@ void makeFlakeClosureGCRoot(Store & store, store.addIndirectRoot(symlink); } -struct InstallableFlake : InstallableValue +std::vector InstallableFlake::getActualAttrPaths() { - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; + std::vector res; - InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) - : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) - { } + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); - InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, - std::string attrPath, Strings && prefixes) - : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, - prefixes(prefixes) - { } + for (auto & s : attrPaths) + res.push_back(s); - std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } + return res; +} - std::vector getActualAttrPaths() - { - std::vector res; +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake) +{ + auto vFlake = state.allocValue(); - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); + callFlake(state, resFlake, *vFlake); - for (auto & s : attrPaths) - res.push_back(s); + makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); - return res; - } + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); - Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake) - { - auto vFlake = state.allocValue(); + state.forceValue(*(*aOutputs)->value); - callFlake(state, resFlake, *vFlake); + return (*aOutputs)->value; +} - makeFlakeClosureGCRoot(*state.store, flakeRef, resFlake); +std::vector InstallableFlake::toDerivations() +{ + auto state = cmd.getEvalState(); - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); + auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode()); - state.forceValue(*(*aOutputs)->value); + Value * vOutputs = nullptr; - return (*aOutputs)->value; - } + auto emptyArgs = state->allocBindings(0); - std::vector toDerivations() override - { - auto state = cmd.getEvalState(); + auto & evalCache = flake::EvalCache::singleton(); - auto resFlake = resolveFlake(*state, flakeRef, cmd.getLockFileMode()); + auto fingerprint = resFlake.getFingerprint(); - Value * vOutputs = nullptr; - - auto emptyArgs = state->allocBindings(0); - - auto & evalCache = flake::EvalCache::singleton(); - - auto fingerprint = resFlake.getFingerprint(); - - for (auto & attrPath : getActualAttrPaths()) { - auto drv = evalCache.getDerivation(fingerprint, attrPath); - if (drv) { - if (state->store->isValidPath(drv->drvPath)) - return {*drv}; - } - - if (!vOutputs) - vOutputs = getFlakeOutputs(*state, resFlake); - - try { - auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs); - state->forceValue(*v); - - auto drvInfo = getDerivation(*state, *v, false); - if (!drvInfo) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drv = flake::EvalCache::Derivation{ - drvInfo->queryDrvPath(), - drvInfo->queryOutPath(), - drvInfo->queryOutputName() - }; - - evalCache.addDerivation(fingerprint, attrPath, drv); - - return {drv}; - } catch (AttrPathNotFound & e) { - } + for (auto & attrPath : getActualAttrPaths()) { + auto drv = evalCache.getDerivation(fingerprint, attrPath); + if (drv) { + if (state->store->isValidPath(drv->drvPath)) + return {*drv}; } - throw Error("flake '%s' does not provide attribute %s", - flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); - } + if (!vOutputs) + vOutputs = getFlakeOutputs(*state, resFlake); - Value * toValue(EvalState & state) override - { - auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + try { + auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs); + state->forceValue(*v); - auto vOutputs = getFlakeOutputs(state, resFlake); + auto drvInfo = getDerivation(*state, *v, false); + if (!drvInfo) + throw Error("flake output attribute '%s' is not a derivation", attrPath); - auto emptyArgs = state.allocBindings(0); + auto drv = flake::EvalCache::Derivation{ + drvInfo->queryDrvPath(), + drvInfo->queryOutPath(), + drvInfo->queryOutputName() + }; - for (auto & attrPath : getActualAttrPaths()) { - try { - auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); - state.forceValue(*v); - return v; - } catch (AttrPathNotFound & e) { - } + evalCache.addDerivation(fingerprint, attrPath, drv); + + return {drv}; + } catch (AttrPathNotFound & e) { } - - throw Error("flake '%s' does not provide attribute %s", - flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } -}; + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); +} + +Value * InstallableFlake::toValue(EvalState & state) +{ + auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); + + auto vOutputs = getFlakeOutputs(state, resFlake); + + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { + try { + auto * v = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); + state.forceValue(*v); + return v; + } catch (AttrPathNotFound & e) { + } + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); +} // FIXME: extend std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000..020a61a2b --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,97 @@ +#pragma once + +#include "util.hh" +#include "flake/eval-cache.hh" + +#include + +namespace nix { + +struct Value; +class EvalState; +class SourceExprCommand; + +struct Buildable +{ + Path drvPath; // may be empty + std::map outputs; +}; + +typedef std::vector Buildables; + +struct App +{ + PathSet context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... + + App(EvalState & state, Value & vApp); +}; + +struct Installable +{ + virtual ~Installable() { } + + virtual std::string what() = 0; + + virtual Buildables toBuildables() + { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + App toApp(EvalState & state); + + virtual Value * toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } + + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional getStorePath() + { + return {}; + } +}; + +struct InstallableValue : Installable +{ + SourceExprCommand & cmd; + + InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + + virtual std::vector toDerivations(); + + Buildables toBuildables() override; +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths)) + { } + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, + std::string attrPath, Strings && prefixes) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath}, + prefixes(prefixes) + { } + + std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } + + std::vector getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake); + + std::vector toDerivations() override; + + Value * toValue(EvalState & state) override; +}; + +} From 7d38060a0da2698052e84a0cfee422d409a38187 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Oct 2019 17:52:10 +0200 Subject: [PATCH 0562/2548] Support non-x86_64-linux system types in flakes A command like $ nix run nixpkgs#hello will now build the attribute 'packages.${system}.hello' rather than 'packages.hello'. Note that this does mean that the flake needs to export an attribute for every system type it supports, and you can't build on unsupported systems. So 'packages' typically looks like this: packages = nixpkgs.lib.genAttrs ["x86_64-linux" "i686-linux"] (system: { hello = ...; }); The 'checks', 'defaultPackage', 'devShell', 'apps' and 'defaultApp' outputs similarly are now attrsets that map system types to derivations/apps. 'nix flake check' checks that the derivations for all platforms evaluate correctly, but only builds the derivations in 'checks.${system}'. Fixes #2861. (That issue also talks about access to ~/.config/nixpkgs and --arg, but I think it's reasonable to say that flakes shouldn't support those.) The alternative to attribute selection is to pass the system type as an argument to the flake's 'outputs' function, e.g. 'outputs = { self, nixpkgs, system }: ...'. However, that approach would be at odds with hermetic evaluation and make it impossible to enumerate the packages provided by a flake. --- flake.nix | 26 +++++++------- src/nix/command.hh | 17 ++------- src/nix/flake.cc | 80 +++++++++++++++++++++++++++++++---------- src/nix/installables.cc | 17 +++++++++ src/nix/run.cc | 4 +-- src/nix/shell.cc | 2 +- tests/flakes.sh | 24 ++++++------- 7 files changed, 109 insertions(+), 61 deletions(-) diff --git a/flake.nix b/flake.nix index 4ec80e9d6..2f0073cc8 100644 --- a/flake.nix +++ b/flake.nix @@ -13,8 +13,10 @@ systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = nixpkgs.lib.genAttrs systems (system: + nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; overlays = [ self.overlay ]; @@ -427,19 +429,19 @@ }; - checks = { - binaryTarball = self.hydraJobs.binaryTarball.x86_64-linux; - perlBindings = self.hydraJobs.perlBindings.x86_64-linux; - }; + checks = forAllSystems (system: { + binaryTarball = self.hydraJobs.binaryTarball.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; + }); - packages = { - inherit (nixpkgsFor.x86_64-linux) nix; - }; + packages = forAllSystems (system: { + inherit (nixpkgsFor.${system}) nix; + }); - defaultPackage = self.packages.nix; + defaultPackage = forAllSystems (system: self.packages.${system}.nix); - devShell = - with nixpkgsFor.x86_64-linux; + devShell = forAllSystems (system: + with nixpkgsFor.${system}; with commonDeps pkgs; stdenv.mkDerivation { @@ -461,7 +463,7 @@ PATH=$prefix/bin:$PATH unset PYTHONPATH ''; - }; + }); }; } diff --git a/src/nix/command.hh b/src/nix/command.hh index 802dd9828..93f324071 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -64,22 +64,9 @@ struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions std::shared_ptr parseInstallable( ref store, const std::string & installable); - virtual Strings getDefaultFlakeAttrPaths() - { - return {"defaultPackage"}; - } + virtual Strings getDefaultFlakeAttrPaths(); - virtual Strings getDefaultFlakeAttrPathPrefixes() - { - return { - // As a convenience, look for the attribute in - // 'outputs.packages'. - "packages.", - // As a temporary hack until Nixpkgs is properly converted - // to provide a clean 'packages' set, look in 'legacyPackages'. - "legacyPackages." - }; - } + virtual Strings getDefaultFlakeAttrPathPrefixes(); }; enum RealiseMode { Build, NoBuild, DryRun }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e6dc5680f..d928af3b9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -251,6 +251,12 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = resolveFlake(); + auto checkSystemName = [&](const std::string & system, const Pos & pos) { + // FIXME: what's the format of "system"? + if (system.find('-') == std::string::npos) + throw Error("'%s' is not a valid system type, at %s", system, pos); + }; + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { auto drvInfo = getDerivation(*state, v, false); @@ -374,34 +380,70 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON if (name == "checks") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - drvPaths.insert(checkDerivation( - name + "." + (std::string) attr.name, *attr.value, *attr.pos)); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + if ((std::string) attr.name == settings.thisSystem.get()) + drvPaths.insert(drvPath); + } + } } else if (name == "packages") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - checkDerivation( - name + "." + (std::string) attr.name, *attr.value, *attr.pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } } else if (name == "apps") { state->forceAttrs(vOutput, pos); - for (auto & attr : *vOutput.attrs) - checkApp( - name + "." + (std::string) attr.name, *attr.value, *attr.pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } } - else if (name == "defaultPackage" || name == "devShell") - checkDerivation(name, vOutput, pos); + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkDerivation( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } - else if (name == "defaultApp") - checkApp(name, vOutput, pos); + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } - else if (name == "legacyPackages") - // FIXME: do getDerivations? - ; + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + // FIXME: do getDerivations? + } + } else if (name == "overlay") checkOverlay(name, vOutput, pos); @@ -409,7 +451,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON else if (name == "overlays") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) - checkOverlay(name + "." + (std::string) attr.name, + checkOverlay(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); } @@ -419,14 +461,14 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON else if (name == "nixosModules") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) - checkModule(name + "." + (std::string) attr.name, + checkModule(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); } else if (name == "nixosConfigurations") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) - checkNixOSConfiguration(name + "." + (std::string) attr.name, + checkNixOSConfiguration(fmt("%s.%s", name, attr.name), *attr.value, *attr.pos); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 93509955a..bc8fbeb8f 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -55,6 +55,23 @@ SourceExprCommand::SourceExprCommand() .dest(&file); } +Strings SourceExprCommand::getDefaultFlakeAttrPaths() +{ + return {"defaultPackage." + settings.thisSystem.get()}; +} + +Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() +{ + return { + // As a convenience, look for the attribute in + // 'outputs.packages'. + "packages." + settings.thisSystem.get() + ".", + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + "legacyPackages." + settings.thisSystem.get() + "." + }; +} + ref EvalCommand::getEvalState() { if (!evalState) { diff --git a/src/nix/run.cc b/src/nix/run.cc index 01ec9a6f8..d444fd2eb 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -215,12 +215,12 @@ struct CmdApp : InstallableCommand, RunCommon Strings getDefaultFlakeAttrPaths() override { - return {"defaultApp"}; + return {"defaultApp." + settings.thisSystem.get()}; } Strings getDefaultFlakeAttrPathPrefixes() override { - return {"apps."}; + return {"apps." + settings.thisSystem.get() + "."}; } void run(ref store) override diff --git a/src/nix/shell.cc b/src/nix/shell.cc index a4488b229..50d0f9c88 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -198,7 +198,7 @@ struct Common : InstallableCommand, MixProfile Strings getDefaultFlakeAttrPaths() override { - return {"devShell", "defaultPackage"}; + return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; } Path getShellOutPath(ref store) diff --git a/tests/flakes.sh b/tests/flakes.sh index c5e19826c..73f9d2685 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -34,8 +34,8 @@ cat > $flake1Dir/flake.nix < $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < Date: Tue, 15 Oct 2019 19:53:29 +0200 Subject: [PATCH 0563/2548] Fix 'nix flake init' --- src/nix/flake-template.nix | 10 +++------- tests/flakes.sh | 7 ++++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/nix/flake-template.nix b/src/nix/flake-template.nix index bec613f6c..eb8eb14fc 100644 --- a/src/nix/flake-template.nix +++ b/src/nix/flake-template.nix @@ -1,15 +1,11 @@ { - name = "hello"; - description = "A flake for building Hello World"; - epoch = 201906; + edition = 201909; - requires = [ "nixpkgs" ]; + outputs = { self, nixpkgs }: { - provides = deps: rec { - - packages.hello = deps.nixpkgs.provides.packages.hello; + packages.x86_64-linux.hello = nixpkgs.packages.x86_64-linux.hello; }; } diff --git a/tests/flakes.sh b/tests/flakes.sh index 73f9d2685..6cae8029c 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -36,6 +36,9 @@ cat > $flake1Dir/flake.nix < Date: Wed, 16 Oct 2019 00:12:40 +0200 Subject: [PATCH 0564/2548] Improve GitHub caching In particular, when building a flake lock file, inputs like 'nixpkgs' are now downloaded only once. Previously, it would fetch https://api.github.com/repos///tarball/ and then later https://api.github.com/repos///tarball/, even though they produce the same result. Git and GitHub now also share a cache that maps revs to a store path and other info. --- src/libexpr/flake/flake.cc | 37 +------- src/libexpr/primops/fetchGit.cc | 157 +++++++++++++++++++++++++------- src/libexpr/primops/fetchGit.hh | 15 ++- 3 files changed, 136 insertions(+), 73 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index e8eb353fb..5fb40fabd 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -170,43 +170,10 @@ static SourceInfo fetchInput(EvalState & state, const FlakeRef & resolvedRef) // This only downloads only one revision of the repo, not the entire history. if (auto refData = std::get_if(&resolvedRef.data)) { - - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. - - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - refData->owner, refData->repo, - resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) - : resolvedRef.ref ? *resolvedRef.ref : "master"); - - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - CachedDownloadRequest request(url); - request.unpack = true; - request.name = "source"; - request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; - request.getLastModified = true; - auto result = getDownloader()->downloadCached(state.store, request); - - if (!result.etag) - throw Error("did not receive an ETag header from '%s'", url); - - if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') - throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - - FlakeRef ref(resolvedRef.baseRef()); - ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); - SourceInfo info(ref); - info.storePath = result.storePath; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - info.lastModified = result.lastModified; - - return info; + return doGit(exportGitHub(state.store, refData->owner, refData->repo, resolvedRef.ref, resolvedRef.rev)); } - // This downloads the entire git history + // This downloads the entire git history. else if (auto refData = std::get_if(&resolvedRef.data)) { return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 21fa025c1..50277672c 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -18,6 +18,60 @@ namespace nix { extern std::regex revRegex; +static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) +{ + Path cacheDir = getCacheDir() + "/nix/git-revs"; + std::string linkName = + name == "source" + ? rev.gitRev() + : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false); + return cacheDir + "/" + linkName + ".link"; +} + +static void cacheGitInfo(const std::string & name, const GitInfo & gitInfo) +{ + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["name"] = name; + json["rev"] = gitInfo.rev.gitRev(); + if (gitInfo.revCount) + json["revCount"] = *gitInfo.revCount; + json["lastModified"] = gitInfo.lastModified; + + auto cacheInfoPath = getCacheInfoPathFor(name, gitInfo.rev); + createDirs(dirOf(cacheInfoPath)); + writeFile(cacheInfoPath, json.dump()); +} + +static std::optional lookupGitInfo( + ref store, + const std::string & name, + const Hash & rev) +{ + try { + auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev))); + + assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev); + + Path storePath = json["storePath"]; + + if (store->isValidPath(storePath)) { + GitInfo gitInfo; + gitInfo.storePath = storePath; + gitInfo.rev = rev; + if (json.find("revCount") != json.end()) + gitInfo.revCount = json["revCount"]; + gitInfo.lastModified = json["lastModified"]; + return gitInfo; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + return {}; +} + GitInfo exportGit(ref store, std::string uri, std::optional ref, std::optional rev, @@ -25,6 +79,17 @@ GitInfo exportGit(ref store, std::string uri, { assert(!rev || rev->type == htSHA1); + if (rev) { + if (auto gitInfo = lookupGitInfo(store, name, *rev)) { + // If this gitInfo was produced by exportGitHub, then it won't + // have a revCount. So we have to do a full clone. + if (gitInfo->revCount) { + gitInfo->ref = ref; + return *gitInfo; + } + } + } + if (hasPrefix(uri, "git+")) uri = std::string(uri, 4); bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git"); @@ -100,9 +165,6 @@ GitInfo exportGit(ref store, std::string uri, isLocal = true; } - deletePath(getCacheDir() + "/nix/git"); - deletePath(getCacheDir() + "/nix/gitv2"); - Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, uri).to_string(Base32, false); Path repoDir; @@ -179,6 +241,13 @@ GitInfo exportGit(ref store, std::string uri, rev = Hash(chomp(readFile(localRefFile)), htSHA1); } + if (auto gitInfo = lookupGitInfo(store, name, *rev)) { + if (gitInfo->revCount) { + gitInfo->ref = ref; + return *gitInfo; + } + } + // FIXME: check whether rev is an ancestor of ref. GitInfo gitInfo; gitInfo.ref = *ref; @@ -186,29 +255,6 @@ GitInfo exportGit(ref store, std::string uri, printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, - name + std::string("\0"s) + gitInfo.rev.gitRev()).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == gitInfo.rev); - - Path storePath = json["storePath"]; - - if (store->isValidPath(storePath)) { - gitInfo.storePath = storePath; - gitInfo.revCount = json["revCount"]; - gitInfo.lastModified = json["lastModified"]; - return gitInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - // FIXME: should pipe this, or find some better way to extract a // revision. auto tar = runProgram("git", true, { "-C", repoDir, "archive", gitInfo.rev.gitRev() }); @@ -223,15 +269,55 @@ GitInfo exportGit(ref store, std::string uri, gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "show", "-s", "--format=%ct", gitInfo.rev.gitRev() })); - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev.gitRev(); - json["revCount"] = gitInfo.revCount; - json["lastModified"] = gitInfo.lastModified; + cacheGitInfo(name, gitInfo); - writeFile(storeLink, json.dump()); + return gitInfo; +} + +GitInfo exportGitHub( + ref store, + const std::string & owner, + const std::string & repo, + std::optional ref, + std::optional rev) +{ + if (rev) { + if (auto gitInfo = lookupGitInfo(store, "source", *rev)) + return *gitInfo; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev ? rev->to_string(Base16, false) : ref ? *ref : "master"); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; + auto result = getDownloader()->downloadCached(store, request); + + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + assert(result.lastModified); + + GitInfo gitInfo; + gitInfo.storePath = result.storePath; + gitInfo.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + gitInfo.lastModified = *result.lastModified; + + // FIXME: this can overwrite a cache file that contains a revCount. + cacheGitInfo("source", gitInfo); return gitInfo; } @@ -283,7 +369,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.rev.gitShortRev()); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + assert(gitInfo.revCount); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *gitInfo.revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchGit.hh b/src/libexpr/primops/fetchGit.hh index 006fa8b5f..fe2b49942 100644 --- a/src/libexpr/primops/fetchGit.hh +++ b/src/libexpr/primops/fetchGit.hh @@ -9,15 +9,24 @@ namespace nix { struct GitInfo { Path storePath; - std::string ref; + std::optional ref; Hash rev{htSHA1}; - uint64_t revCount; + std::optional revCount; time_t lastModified; }; -GitInfo exportGit(ref store, std::string uri, +GitInfo exportGit( + ref store, + std::string uri, std::optional ref, std::optional rev, const std::string & name); +GitInfo exportGitHub( + ref store, + const std::string & owner, + const std::string & repo, + std::optional ref, + std::optional rev); + } From a56036fa872d86b09586c41ead475d537b6df0a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Oct 2019 00:20:18 +0200 Subject: [PATCH 0565/2548] Fix repeated fetchGit.sh test --- tests/fetchGit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index be46d24a7..885dd9e77 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -11,7 +11,7 @@ repo=$TEST_ROOT/git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv* git init $repo git -C $repo config user.email "foobar@example.com" From 8e478c234100cf03ea1b777d4bd42a9be7be9e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Oct 2019 17:45:09 +0200 Subject: [PATCH 0566/2548] Add experimental-features setting Experimental features are now opt-in. There are currently two experimental features: "nix-command" (which enables the "nix" command), and "flakes" (which enables support for flakes). This will allow us to merge experimental features more quickly, without committing to supporting them indefinitely. Typical usage: $ nix build --experimental-features 'nix-command flakes' nixpkgs#hello --- src/libexpr/flake/flake.cc | 2 ++ src/libstore/globals.cc | 7 +++++++ src/libstore/globals.hh | 5 +++++ src/nix/main.cc | 2 ++ tests/init.sh | 1 + 5 files changed, 17 insertions(+) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5fb40fabd..d03227f69 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -433,6 +433,8 @@ static std::pair updateLocks( and optionally write it to file, it the flake is writable. */ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) { + settings.requireExperimentalFeature("flakes"); + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1c2c08715..249c36673 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +void Settings::requireExperimentalFeature(const std::string & name) +{ + auto & f = experimentalFeatures.get(); + if (std::find(f.begin(), f.end(), name) == f.end()) + throw Error("experimental Nix feature '%s' is disabled", name); +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c0c535a12..0b1a8dac5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -356,6 +356,11 @@ public: Setting githubAccessToken{this, "", "github-acces-token", "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + + Setting experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + void requireExperimentalFeature(const std::string & name); }; diff --git a/src/nix/main.cc b/src/nix/main.cc index eedf8656e..8cdeed8f5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -149,6 +149,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + settings.requireExperimentalFeature("nix-command"); + initPlugins(); if (!args.command) args.showHelpAndExit(); diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..c62c4856a 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,6 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < Date: Sat, 12 Oct 2019 13:00:19 +0000 Subject: [PATCH 0567/2548] Downloader: Warn if no trusted CA file has been configured --- src/libstore/download.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/download.cc b/src/libstore/download.cc index cdf56e09d..e5f8556b7 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -285,6 +285,7 @@ struct CurlDownloader : public Downloader } if (request.verifyTLS) { + debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); } else { From f0ec4b4ce478a8d2760203e8192275b88c770e1c Mon Sep 17 00:00:00 2001 From: Steven Shaw Date: Sat, 19 Oct 2019 13:25:18 +1000 Subject: [PATCH 0568/2548] Fix unset variable in installer --- scripts/install-nix-from-closure.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 35926f3da..3f1581854 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -141,11 +141,9 @@ if [ -z "$_NIX_INSTALLER_TEST" ]; then fi added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then - # Make the shell source nix.sh during login. - p=$HOME/.nix-profile/etc/profile.d/nix.sh - for i in .bash_profile .bash_login .profile; do fn="$HOME/$i" if [ -w "$fn" ]; then @@ -157,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done - fi if [ -z "$added" ]; then From 8426d99b0efc0b22a01db0e2e97adb72134ac5bf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 20 Oct 2019 16:43:00 +0200 Subject: [PATCH 0569/2548] Fix InstallableFlake::what() --- src/nix/installables.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 020a61a2b..a635cb96f 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -83,7 +83,7 @@ struct InstallableFlake : InstallableValue prefixes(prefixes) { } - std::string what() override { return flakeRef.to_string() + ":" + *attrPaths.begin(); } + std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } std::vector getActualAttrPaths(); From d8730fb86facadbef22d3df7f8a743a56e7ed53c Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Sun, 20 Oct 2019 19:08:05 -0700 Subject: [PATCH 0570/2548] Document import syntax --- doc/manual/expressions/builtins.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..21da8b87b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,13 +733,21 @@ builtins.genList (x: x * x) 5 import path + import + <path> builtins.import path + builtins.import + <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. Evaluation aborts if the + in that directory is loaded. If the + <path> syntax is used, the path will be resolved + relative to those listed in the NIX_PATH environment + variable (see the section on NIX_PATH + for details on how the resolution works). Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From 389a2cebed7cd72bda524ece0a56af2888cd80b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 13:14:39 +0200 Subject: [PATCH 0571/2548] SourceExprCommand::getSourceExpr(): Allocate more space Fixes #3140. --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 52f9778fc..0e8bba39d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -39,7 +39,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, searchPath.size() + 1); + state.mkAttrs(*vSourceExpr, 1024); mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); From aabf5c86c9df1b4e1616a4cf06ee14a6edf2e5e1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Oct 2019 17:45:09 +0200 Subject: [PATCH 0572/2548] Add experimental-features setting Experimental features are now opt-in. There is currently one experimental feature: "nix-command" (which enables the "nix" command. This will allow us to merge experimental features more quickly, without committing to supporting them indefinitely. Typical usage: $ nix build --experimental-features 'nix-command flakes' nixpkgs#hello (cherry picked from commit 8e478c234100cf03ea1b777d4bd42a9be7be9e8c, without the "flakes" feature) --- src/libstore/globals.cc | 7 +++++++ src/libstore/globals.hh | 5 +++++ src/nix/main.cc | 2 ++ tests/init.sh | 1 + 4 files changed, 15 insertions(+) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1c2c08715..249c36673 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +void Settings::requireExperimentalFeature(const std::string & name) +{ + auto & f = experimentalFeatures.get(); + if (std::find(f.begin(), f.end(), name) == f.end()) + throw Error("experimental Nix feature '%s' is disabled", name); +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ab1c09aa2..1221e4db7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -353,6 +353,11 @@ public: Setting pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + void requireExperimentalFeature(const std::string & name); }; diff --git a/src/nix/main.cc b/src/nix/main.cc index a80fd0ea6..22f5e8c4c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -138,6 +138,8 @@ void mainWrapped(int argc, char * * argv) args.parseCmdline(argvToStrings(argc, argv)); + settings.requireExperimentalFeature("nix-command"); + initPlugins(); if (!args.command) args.showHelpAndExit(); diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..6a119aad0 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,6 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < Date: Mon, 21 Oct 2019 14:57:01 +0200 Subject: [PATCH 0573/2548] Don't ignore revs/refs of local flakerefs Fixes error: the content hash of flake '/home/eelco/Dev/nixpkgs-flake?ref=HEAD&rev=0000000000000000000000000000000000000000' doesn't match the hash recorded in the referring lockfile --- src/libexpr/flake/flake.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index d03227f69..73ca59452 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -181,7 +181,7 @@ static SourceInfo fetchInput(EvalState & state, const FlakeRef & resolvedRef) else if (auto refData = std::get_if(&resolvedRef.data)) { if (!pathExists(refData->path + "/.git")) throw Error("flake '%s' does not reference a Git repository", refData->path); - return doGit(exportGit(state.store, refData->path, {}, {}, "source")); + return doGit(exportGit(state.store, refData->path, resolvedRef.ref, resolvedRef.rev, "source")); } else abort(); @@ -448,11 +448,15 @@ ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLoc + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); } + debug("old lock file: %s", oldLockFile); + RefMap refMap; LockFile lockFile(updateLocks( refMap, "", state, flake, handleLockFile, oldLockFile, true).second); + debug("new lock file: %s", lockFile); + if (!(lockFile == oldLockFile)) { if (allowedToWrite(handleLockFile)) { if (auto refData = std::get_if(&topRef.data)) { @@ -524,7 +528,8 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V auto flake = getFlake(state, lazyInput->lockedInput.ref, false); if (flake.sourceInfo.narHash != lazyInput->lockedInput.narHash) - throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", + lazyInput->lockedInput.ref); callFlake(state, flake, lazyInput->lockedInput, v); } else { @@ -532,7 +537,8 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V auto sourceInfo = getNonFlake(state, lazyInput->lockedInput.ref, false, refMap); if (sourceInfo.narHash != lazyInput->lockedInput.narHash) - throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", sourceInfo.resolvedRef); + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", + lazyInput->lockedInput.ref); state.mkAttrs(v, 8); From a7aabd7cc785bbf34ad29101672677ced18a7fdd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 16:07:19 +0200 Subject: [PATCH 0574/2548] Add getDefaultProfile() function --- src/libstore/profiles.cc | 18 ++++++++++++++++++ src/libstore/profiles.hh | 4 ++++ src/nix-env/nix-env.cc | 17 ++--------------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4c6af567a..29f6f6c17 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 5fa1533de..78645d8b6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 199dc92aa..5ac0eb87c 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1427,21 +1427,8 @@ static int _main(int argc, char * * argv) if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE", ""); - if (globals.profile == "") { - Path profileLink = getHome() + "/.nix-profile"; - try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); - } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); - } catch (Error &) { - globals.profile = profileLink; - } - } + if (globals.profile == "") + globals.profile = getDefaultProfile(); op(globals, opFlags, opArgs); From b82f75464d1e5ae9a00d8004e5dd7b1ca05059e4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:17:15 +0200 Subject: [PATCH 0575/2548] buildenv: Eliminate global variables, other cleanup --- src/libstore/build.cc | 1 + src/libstore/builtins.hh | 1 - src/libstore/builtins/buildenv.cc | 128 ++++++++++++++---------------- src/libstore/builtins/buildenv.hh | 21 +++++ 4 files changed, 81 insertions(+), 70 deletions(-) create mode 100644 src/libstore/builtins/buildenv.hh diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cdf848c98..1b27d7af0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "builtins/buildenv.hh" #include "download.hh" #include "finally.hh" #include "compression.hh" diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 0d2da873e..f9b5f7900 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,6 +6,5 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 096593886..c1c85d0bf 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,4 @@ -#include "builtins.hh" +#include "buildenv.hh" #include #include @@ -7,16 +7,14 @@ namespace nix { -typedef std::map Priorities; - -// FIXME: change into local variables. - -static Priorities priorities; - -static unsigned long symlinks; +struct State +{ + std::map priorities; + unsigned long symlinks = 0; +}; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { DirEntries srcFiles; @@ -67,7 +65,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); + createLinks(state, srcFile, dstFile, priority); continue; } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); @@ -77,8 +75,8 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) throw SysError(format("unlinking '%1%'") % dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) @@ -90,7 +88,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; + auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( "packages '%1%' and '%2%' have the same priority %3%; " @@ -109,67 +107,30 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) } createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + state.priorities[dstFile] = priority; + state.symlinks++; } } -typedef std::set FileProp; - -static FileProp done; -static FileProp postponed = FileProp{}; - -static Path out; - -static void addPkg(const Path & pkgDir, int priority) +void buildProfile(const Path & out, Packages && pkgs) { - if (!done.insert(pkgDir).second) return; - createLinks(pkgDir, out, priority); + State state; - try { - for (const auto & p : tokenizeString>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } -} + std::set done, postponed; -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} -}; + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); -typedef std::vector Packages; - -void builtinBuildenv(const BasicDerivation & drv) -{ - auto getAttr = [&](const string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; - - out = getAttr("out"); - createDirs(out); - - /* Convert the stuff we get from the environment back into a - * coherent data type. */ - Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; } - } + }; /* Symlink to the packages that have been installed explicitly by the * user. Process in priority order to reduce unnecessary @@ -189,13 +150,42 @@ void builtinBuildenv(const BasicDerivation & drv) */ auto priorityCounter = 1000; while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; + std::set pkgDirs; + postponed.swap(pkgDirs); for (const auto & pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", symlinks); + printError("created %d symlinks in user environment", state.symlinks); +} + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + Path out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + buildProfile(out, std::move(pkgs)); createSymlink(getAttr("manifest"), out + "/manifest.nix"); } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 000000000..0a37459b0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} From 0abb3ad53795aa3a4792d30e5721a337f0eddfb7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Mar 2018 00:56:13 +0200 Subject: [PATCH 0576/2548] Allow content-addressable paths to have references This adds a command 'nix make-content-addressable' that rewrites the specified store paths into content-addressable paths. The advantage of such paths is that 1) they can be imported without signatures; 2) they can enable deduplication in cases where derivation changes do not cause output changes (apart from store path hashes). For example, $ nix make-content-addressable -r nixpkgs.cowsay rewrote '/nix/store/g1g31ah55xdia1jdqabv1imf6mcw0nb1-glibc-2.25-49' to '/nix/store/48jfj7bg78a8n4f2nhg269rgw1936vj4-glibc-2.25-49' ... rewrote '/nix/store/qbi6rzpk0bxjw8lw6azn2mc7ynnn455q-cowsay-3.03+dfsg1-16' to '/nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16' We can then copy the resulting closure to another store without signatures: $ nix copy --trusted-public-keys '' ---to ~/my-nix /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 In order to support self-references in content-addressable paths, these paths are hashed "modulo" self-references, meaning that self-references are zeroed out during hashing. Somewhat annoyingly, this means that the NAR hash stored in the Nix database is no longer necessarily equal to the output of "nix hash-path"; for content-addressable paths, you need to pass the --modulo flag: $ nix path-info --json /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 | jq -r .[].narHash sha256:0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 1ggznh07khq0hz6id09pqws3a8q9pn03ya3c03nwck1kwq8rclzs $ nix hash-path --type sha256 --base32 /nix/store/iq6g2x4q62xp7y7493bibx0qn5w7xz67-cowsay-3.03+dfsg1-16 --modulo iq6g2x4q62xp7y7493bibx0qn5w7xz67 0ri611gdilz2c9rsibqhsipbfs9vwcqvs811a52i2bnkhv7w9mgw --- src/libstore/build.cc | 19 +------ src/libstore/local-store.cc | 21 ++++++-- src/libstore/references.cc | 62 ++++++++++++++++++++++ src/libstore/references.hh | 29 +++++++++- src/libstore/store-api.cc | 38 +++++++------ src/libstore/store-api.hh | 3 +- src/libutil/hash.cc | 20 ++----- src/libutil/hash.hh | 11 ++-- src/libutil/serialise.hh | 2 +- src/libutil/util.cc | 13 +++++ src/libutil/util.hh | 14 +++++ src/nix/command.cc | 2 +- src/nix/command.hh | 4 ++ src/nix/hash.cc | 22 +++++++- src/nix/make-content-addressable.cc | 82 +++++++++++++++++++++++++++++ src/nix/verify.cc | 12 +++-- 16 files changed, 289 insertions(+), 65 deletions(-) create mode 100644 src/nix/make-content-addressable.cc diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e0f8a545..68c2f2ce3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -727,23 +727,6 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map StringRewrites; - - -std::string rewriteStrings(std::string s, const StringRewrites & rewrites) -{ - for (auto & i : rewrites) { - size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) - s.replace(j, i.first.size(), i.second); - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -865,7 +848,7 @@ private: #endif /* Hash rewriting. */ - StringRewrites inputRewrites, outputRewrites; + StringMap inputRewrites, outputRewrites; typedef map RedirectedOutputs; RedirectedOutputs redirectedOutputs; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d86f7b1d2..1280ecab5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,6 +5,7 @@ #include "worker-protocol.hh" #include "derivations.hh" #include "nar-info.hh" +#include "references.hh" #include #include @@ -1009,17 +1010,21 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + std::unique_ptr hashSink; + if (info.ca == "") + hashSink = std::make_unique(htSHA256); + else + hashSink = std::make_unique(htSHA256, storePathToHash(info.path)); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); - hashSink(data, n); + (*hashSink)(data, n); return n; }); restorePath(realPath, wrapperSource); - auto hashResult = hashSink.finish(); + auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1241,7 +1246,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of '%1%'") % i); - HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + std::unique_ptr hashSink; + if (info->ca == "") + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(info->path)); + + dumpPath(toRealPath(i), *hashSink); + auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError(format("path '%1%' was modified! " diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 0dcc264c3..605ca9815 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -118,4 +118,66 @@ PathSet scanForReferences(const string & path, } +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : from(from), to(to), nextSink(nextSink) +{ + assert(from.size() == to.size()); +} + +void RewritingSink::operator () (const unsigned char * data, size_t len) +{ + std::string s(prev); + s.append((const char *) data, len); + + size_t j = 0; + while ((j = s.find(from, j)) != string::npos) { + matches.push_back(pos + j); + s.replace(j, from.size(), to); + } + + prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + + auto consumed = s.size() - prev.size(); + + pos += consumed; + + if (consumed) nextSink((unsigned char *) s.data(), consumed); +} + +void RewritingSink::flush() +{ + if (prev.empty()) return; + pos += prev.size(); + nextSink((unsigned char *) prev.data(), prev.size()); + prev.clear(); +} + +HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) + : hashSink(ht) + , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) +{ +} + +void HashModuloSink::operator () (const unsigned char * data, size_t len) +{ + rewritingSink(data, len); +} + +HashResult HashModuloSink::finish() +{ + rewritingSink.flush(); + + /* Hash the positions of the self-references. This ensures that a + NAR with self-references and a NAR with some of the + self-references already zeroed out do not produce a hash + collision. FIXME: proof. */ + for (auto & pos : rewritingSink.matches) { + auto s = fmt("|%d", pos); + hashSink((unsigned char *) s.data(), s.size()); + } + + auto h = hashSink.finish(); + return {h.first, rewritingSink.pos}; +} + } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 013809d12..c38bdd720 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,5 +7,32 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); - + +struct RewritingSink : Sink +{ + std::string from, to, prev; + Sink & nextSink; + uint64_t pos = 0; + + std::vector matches; + + RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + + void operator () (const unsigned char * data, size_t len) override; + + void flush(); +}; + +struct HashModuloSink : AbstractHashSink +{ + HashSink hashSink; + RewritingSink rewritingSink; + + HashModuloSink(HashType ht, const std::string & modulus); + + void operator () (const unsigned char * data, size_t len) override; + + HashResult finish() override; +}; + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index be13fa49a..1a1b24e3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -205,15 +205,27 @@ Path Store::makeOutputPath(const string & id, } -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const +static std::string makeType(string && type, const PathSet & references) { - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, + for (auto & i : references) { + type += ":"; + type += i; + } + return type; +} + + +Path Store::makeFixedOutputPath(bool recursive, + const Hash & hash, const string & name, const PathSet & references) const +{ + if (hash.type == htSHA256 && recursive) { + return makeStorePath(makeType("source", references), hash, name); + } else { + assert(references.empty()); + return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), - name); + hash.to_string(Base16) + ":"), name); + } } @@ -224,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash, /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); + return makeStorePath(makeType("text", references), hash, name); } @@ -785,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - if (references.empty() && - store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + auto refs = references; + replaceInSet(refs, path, std::string("self")); + if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path) return true; else warn(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ba8990755..250e1a4bc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -303,7 +303,8 @@ public: const Hash & hash, const string & name) const; Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const; + const Hash & hash, const string & name, + const PathSet & references = {}) const; Path makeTextPath(const string & name, const Hash & hash, const PathSet & references) const; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1c14ebb18..7caee1da7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s) Hash hashFile(HashType ht, const Path & path) { - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); - - std::vector buf(8192); - ssize_t n; - while ((n = read(fd.get(), buf.data(), buf.size()))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file '%1%'") % path); - update(ht, ctx, buf.data(), n); - } - - finish(ht, ctx, hash.hash); - return hash; + HashSink sink(ht); + readFile(path, sink); + return sink.finish().first; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2dbc3b630..ffa43ecf5 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -111,7 +111,12 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public BufferedSink +struct AbstractHashSink : virtual Sink +{ + virtual HashResult finish() = 0; +}; + +class HashSink : public BufferedSink, public AbstractHashSink { private: HashType ht; @@ -122,8 +127,8 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); + void write(const unsigned char * data, size_t len) override; + HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index a344a5ac7..0120aeecb 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct Sink /* A buffered abstract sink. */ -struct BufferedSink : Sink +struct BufferedSink : virtual Sink { size_t bufSize, bufPos; std::unique_ptr buffer; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6f3bf7ae8..2e416edef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1260,6 +1260,19 @@ string replaceStrings(const std::string & s, } +std::string rewriteStrings(const std::string & _s, const StringMap & rewrites) +{ + auto s = _s; + for (auto & i : rewrites) { + if (i.first == i.second) continue; + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f057fdb2c..3493e80b5 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -362,6 +362,20 @@ string replaceStrings(const std::string & s, const std::string & from, const std::string & to); +std::string rewriteStrings(const std::string & s, const StringMap & rewrites); + + +/* If a set contains 'from', remove it and insert 'to'. */ +template +void replaceInSet(std::set & set, const T & from, const T & to) +{ + auto i = set.find(from); + if (i == set.end()) return; + set.erase(i); + set.insert(to); +} + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); diff --git a/src/nix/command.cc b/src/nix/command.cc index 3d7d582d6..532f331a7 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -129,7 +129,7 @@ void StorePathsCommand::run(ref store) } else { - for (auto & p : toStorePaths(store, NoBuild, installables)) + for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p); if (recursive) { diff --git a/src/nix/command.hh b/src/nix/command.hh index 97a6fee7f..45ad1cd2a 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -139,6 +139,10 @@ private: bool recursive = false; bool all = false; +protected: + + RealiseMode realiseMode = NoBuild; + public: StorePathsCommand(bool recursive = false); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index af4105e28..9c06e6116 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -2,6 +2,8 @@ #include "hash.hh" #include "legacy.hh" #include "shared.hh" +#include "references.hh" +#include "archive.hh" using namespace nix; @@ -13,6 +15,7 @@ struct CmdHash : Command bool truncate = false; HashType ht = htSHA256; std::vector paths; + std::experimental::optional modulus; CmdHash(Mode mode) : mode(mode) { @@ -23,6 +26,11 @@ struct CmdHash : Command mkFlag() .longName("type") .mkHashTypeFlag(&ht); + mkFlag() + .longName("modulo") + .description("compute hash modulo specified string") + .labels({"modulus"}) + .dest(&modulus); expectArgs("paths", &paths); } @@ -41,7 +49,19 @@ struct CmdHash : Command void run() override { for (auto path : paths) { - Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + + std::unique_ptr hashSink; + if (modulus) + hashSink = std::make_unique(ht, *modulus); + else + hashSink = std::make_unique(ht); + + if (mode == mFile) + readFile(path, *hashSink); + else + dumpPath(path, *hashSink); + + Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % h.to_string(base, base == SRI); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc new file mode 100644 index 000000000..14ed06413 --- /dev/null +++ b/src/nix/make-content-addressable.cc @@ -0,0 +1,82 @@ +#include "command.hh" +#include "store-api.hh" +#include "references.hh" + +using namespace nix; + +struct CmdMakeContentAddressable : StorePathsCommand +{ + CmdMakeContentAddressable() + { + realiseMode = Build; + } + + std::string name() override + { + return "make-content-addressable"; + } + + std::string description() override + { + return "test"; + } + + void run(ref store, Paths storePaths) override + { + auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); + + paths.reverse(); + + std::map remappings; + + for (auto & path : paths) { + auto oldInfo = store->queryPathInfo(path); + auto oldHashPart = storePathToHash(path); + auto name = storePathToName(path); + + StringSink sink; + store->narFromPath(path, sink); + + StringMap rewrites; + + ValidPathInfo info; + for (auto & ref : oldInfo->references) { + if (ref == path) + info.references.insert("self"); + else { + auto replacement = get(remappings, ref, ref); + // FIXME: warn about unremapped paths? + info.references.insert(replacement); + if (replacement != ref) + rewrites[storePathToHash(ref)] = storePathToHash(replacement); + } + } + + *sink.s = rewriteStrings(*sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + + info.narHash = hashModuloSink.finish().first; + info.narSize = sink.s->size(); + replaceInSet(info.references, path, std::string("self")); + info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references); + replaceInSet(info.references, std::string("self"), info.path); + info.ca = makeFixedOutputCA(true, info.narHash); + + printError("rewrote '%s' to '%s'", path, info.path); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + + store->addToStore(info, *source); + + remappings[path] = info.path; + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 74d9673b6..4b0f80c62 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" +#include "references.hh" #include @@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - HashSink sink(info->narHash.type); - store->narFromPath(info->path, sink); + std::unique_ptr hashSink; + if (info->ca == "") + hashSink = std::make_unique(info->narHash.type); + else + hashSink = std::make_unique(info->narHash.type, storePathToHash(info->path)); - auto hash = sink.finish(); + store->narFromPath(info->path, *hashSink); + + auto hash = hashSink->finish(); if (hash.first != info->narHash) { corrupted++; From d77970fde7fac66cf8c5cdb7688906ba2f5d5362 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:49:16 +0200 Subject: [PATCH 0577/2548] Fix build --- src/nix/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9c06e6116..53a9fd204 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -15,7 +15,7 @@ struct CmdHash : Command bool truncate = false; HashType ht = htSHA256; std::vector paths; - std::experimental::optional modulus; + std::optional modulus; CmdHash(Mode mode) : mode(mode) { From e68736936a11c76725915e628b8d41d9351bbcf2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 17:58:17 +0200 Subject: [PATCH 0578/2548] nix make-content-addressable: Add examples --- src/nix/make-content-addressable.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 14ed06413..16344ee14 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -18,9 +18,22 @@ struct CmdMakeContentAddressable : StorePathsCommand std::string description() override { - return "test"; + return "rewrite a path or closure to content-addressable form"; } + Examples examples() override + { + return { + Example{ + "To create a content-addressable representation of GNU Hello (but not its dependencies):", + "nix make-content-addressable nixpkgs.hello" + }, + Example{ + "To compute a content-addressable representation of the current NixOS system closure:", + "nix make-content-addressable -r /run/current-system" + }, + }; + } void run(ref store, Paths storePaths) override { auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); From 629b9b0049363e091b76b7f60a8357d9f94733cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 18:05:31 +0200 Subject: [PATCH 0579/2548] Mark content-addressable paths with references as experimental --- src/libstore/local-store.cc | 5 ++++- src/nix/hash.cc | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1280ecab5..49061f27d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1013,8 +1013,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, std::unique_ptr hashSink; if (info.ca == "") hashSink = std::make_unique(htSHA256); - else + else { + if (!info.references.empty()) + settings.requireExperimentalFeature("ca-references"); hashSink = std::make_unique(htSHA256, storePathToHash(info.path)); + } LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 53a9fd204..d7451376c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -26,11 +26,13 @@ struct CmdHash : Command mkFlag() .longName("type") .mkHashTypeFlag(&ht); + #if 0 mkFlag() .longName("modulo") .description("compute hash modulo specified string") .labels({"modulus"}) .dest(&modulus); + #endif expectArgs("paths", &paths); } From cb1a79a96aa0602383f2fe33702f6adeee708922 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 18:58:38 +0200 Subject: [PATCH 0580/2548] Fix build --- src/nix/make-content-addressable.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 16344ee14..5b99b5084 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -11,11 +11,6 @@ struct CmdMakeContentAddressable : StorePathsCommand realiseMode = Build; } - std::string name() override - { - return "make-content-addressable"; - } - std::string description() override { return "rewrite a path or closure to content-addressable form"; @@ -92,4 +87,4 @@ struct CmdMakeContentAddressable : StorePathsCommand } }; -static RegisterCommand r1(make_ref()); +static auto r1 = registerCommand("make-content-addressable"); From 45b740c18b196d0326a94df23d08fa3d68e0863f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 22:11:21 +0200 Subject: [PATCH 0581/2548] Use upstream json_fwd.hpp to speed up compilation --- src/libexpr/flake/lockfile.cc | 2 + src/libexpr/flake/lockfile.hh | 2 +- src/libstore/build.cc | 2 +- src/libstore/parsed-derivations.cc | 6 ++- src/libstore/parsed-derivations.hh | 10 +++-- src/nlohmann/json_fwd.hpp | 66 +++++++++++++++++++++++++++--- 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 2c3f3c93d..5693e57dc 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -1,6 +1,8 @@ #include "lockfile.hh" #include "store-api.hh" +#include + namespace nix::flake { LockedInput::LockedInput(const nlohmann::json & json) diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ab81eac8b..757c37989 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -2,7 +2,7 @@ #include "flakeref.hh" -#include +#include namespace nix { class Store; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 21b641f2c..8e795e555 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2521,7 +2521,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; auto json = *structuredAttrs; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 87be8a24e..5553dd863 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,5 +1,7 @@ #include "parsed-derivations.hh" +#include + namespace nix { ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) @@ -9,13 +11,15 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) { try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); + structuredAttrs = std::make_unique(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); } } } +ParsedDerivation::~ParsedDerivation() { } + std::optional ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 9bde4b4dc..6e67e1665 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,6 +1,6 @@ #include "derivations.hh" -#include +#include namespace nix { @@ -8,15 +8,17 @@ class ParsedDerivation { Path drvPath; BasicDerivation & drv; - std::optional structuredAttrs; + std::unique_ptr structuredAttrs; public: ParsedDerivation(const Path & drvPath, BasicDerivation & drv); - const std::optional & getStructuredAttrs() const + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const { - return structuredAttrs; + return structuredAttrs.get(); } std::optional getStringAttr(const std::string & name) const; diff --git a/src/nlohmann/json_fwd.hpp b/src/nlohmann/json_fwd.hpp index ae6e4c64f..28fd10d45 100644 --- a/src/nlohmann/json_fwd.hpp +++ b/src/nlohmann/json_fwd.hpp @@ -1,10 +1,64 @@ -#pragma once +#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ +#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ -namespace nlohmann { +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector -struct json : basic_json<> +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann { - using basic_json<>::basic_json; -}; +/*! +@brief default JSONSerializer template argument -} +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ From 9be7787ec04d604a4e65a22a6b9ae56567aaf618 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 13:12:41 -0700 Subject: [PATCH 0582/2548] Revert "Document import syntax" This reverts commit d8730fb86facadbef22d3df7f8a743a56e7ed53c. --- doc/manual/expressions/builtins.xml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 21da8b87b..465fa1e0b 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -733,21 +733,13 @@ builtins.genList (x: x * x) 5 import path - import - <path> builtins.import path - builtins.import - <path> Load, parse and return the Nix expression in the file path. If path is a directory, the file default.nix - in that directory is loaded. If the - <path> syntax is used, the path will be resolved - relative to those listed in the NIX_PATH environment - variable (see the section on NIX_PATH - for details on how the resolution works). Evaluation aborts if the + in that directory is loaded. Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. import implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a From b7a936224ee6e1ad9c842f618bbd5bc1864ebd54 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:11:26 -0700 Subject: [PATCH 0583/2548] Add hint about path in builtins.import --- doc/manual/expressions/builtins.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 465fa1e0b..8b53cd578 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -746,6 +746,10 @@ builtins.genList (x: x * x) 5 separate file, and use it from Nix expressions in other files. + Unlike some languages, import is a regular + function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + values (see ). + A Nix expression loaded by import must not contain any free variables (identifiers that are not defined in the Nix expression itself and are not From 1e23b82a5303fa9f22f8943c0c3f3a65bc5eb9a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 23:14:29 +0200 Subject: [PATCH 0584/2548] exportGitHub(): Don't rely on the ETag from GitHub We relied on it being the Git revision, but that stopped being the case. --- src/libexpr/primops/fetchGit.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 50277672c..e8d87ffa4 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -286,11 +286,21 @@ GitInfo exportGitHub( return *gitInfo; } + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref ? *ref : "master"); + CachedDownloadRequest request(url); + request.ttl = rev ? 1000000000 : settings.tarballTtl; + auto result = getDownloader()->downloadCached(store, request); + auto json = nlohmann::json::parse(readFile(result.path)); + rev = Hash(json["sha"], htSHA1); + } + // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev ? rev->to_string(Base16, false) : ref ? *ref : "master"); + owner, repo, rev->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") @@ -299,21 +309,15 @@ GitInfo exportGitHub( CachedDownloadRequest request(url); request.unpack = true; request.name = "source"; - request.ttl = rev ? 1000000000 : settings.tarballTtl; + request.ttl = 1000000000; request.getLastModified = true; auto result = getDownloader()->downloadCached(store, request); - if (!result.etag) - throw Error("did not receive an ETag header from '%s'", url); - - if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') - throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - assert(result.lastModified); GitInfo gitInfo; gitInfo.storePath = result.storePath; - gitInfo.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + gitInfo.rev = *rev; gitInfo.lastModified = *result.lastModified; // FIXME: this can overwrite a cache file that contains a revCount. From f107a270026596e525bdf8df0e2d331783ec4a10 Mon Sep 17 00:00:00 2001 From: Ersin Akinci Date: Mon, 21 Oct 2019 14:16:55 -0700 Subject: [PATCH 0585/2548] Tweak path hint --- doc/manual/expressions/builtins.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 8b53cd578..f39b393bf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -747,7 +747,8 @@ builtins.genList (x: x * x) 5 files. Unlike some languages, import is a regular - function in Nix. Paths using the angle bracket syntax (e.g., import <foo>) are normal + function in Nix. Paths using the angle bracket syntax (e.g., + import <foo>) are normal path values (see ). A Nix expression loaded by import must From 91a88f3acba8978d17ca40fd64e381514a51244d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Oct 2019 23:38:07 +0200 Subject: [PATCH 0586/2548] Fix "nixpkgs." compatibility --- src/nix/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index bc8fbeb8f..38f37adb1 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -409,7 +409,7 @@ std::vector> SourceExprCommand::parseInstallables( bool static warned; warnOnce(warned, "the syntax 'nixpkgs.' is deprecated; use 'nixpkgs:' instead"); result.push_back(std::make_shared(*this, FlakeRef("nixpkgs"), - Strings{"legacyPackages." + std::string(s, 8)})); + Strings{"legacyPackages." + settings.thisSystem.get() + "." + std::string(s, 8)})); } else if ((hash = s.rfind('#')) != std::string::npos) From ce279209363cb6a9cbbe68a13fab9d8550b721f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 00:21:58 +0200 Subject: [PATCH 0587/2548] Add start of 'nix profile' command --- src/nix/command.cc | 5 + src/nix/command.hh | 5 + src/nix/flake.cc | 1 + src/nix/installables.cc | 11 +- src/nix/installables.hh | 3 + src/nix/profile.cc | 234 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 src/nix/profile.cc diff --git a/src/nix/command.cc b/src/nix/command.cc index 57f3754cc..1cb4cc92a 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -123,4 +123,9 @@ void MixProfile::updateProfile(const Buildables & buildables) updateProfile(*result); } +MixDefaultProfile::MixDefaultProfile() +{ + profile = getDefaultProfile(); +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index 546c27a71..ef29381cf 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -188,4 +188,9 @@ struct MixProfile : virtual Args, virtual StoreCommand void updateProfile(const Buildables & buildables); }; +struct MixDefaultProfile : MixProfile +{ + MixDefaultProfile(); +}; + } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index d928af3b9..6e7c5e2eb 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -653,6 +653,7 @@ struct CmdFlake : virtual MultiCommand, virtual Command { if (!command) throw UsageError("'nix flake' requires a sub-command."); + command->prepare(); command->run(); } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 38f37adb1..671cf513a 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -294,7 +294,7 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Resolv return (*aOutputs)->value; } -std::vector InstallableFlake::toDerivations() +std::tuple InstallableFlake::toDerivation() { auto state = cmd.getEvalState(); @@ -312,7 +312,7 @@ std::vector InstallableFlake::toDerivations() auto drv = evalCache.getDerivation(fingerprint, attrPath); if (drv) { if (state->store->isValidPath(drv->drvPath)) - return {*drv}; + return {attrPath, resFlake.flake.sourceInfo.resolvedRef, *drv}; } if (!vOutputs) @@ -334,7 +334,7 @@ std::vector InstallableFlake::toDerivations() evalCache.addDerivation(fingerprint, attrPath, drv); - return {drv}; + return {attrPath, resFlake.flake.sourceInfo.resolvedRef, drv}; } catch (AttrPathNotFound & e) { } } @@ -343,6 +343,11 @@ std::vector InstallableFlake::toDerivations() flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); } +std::vector InstallableFlake::toDerivations() +{ + return {std::get<2>(toDerivation())}; +} + Value * InstallableFlake::toValue(EvalState & state) { auto resFlake = resolveFlake(state, flakeRef, cmd.getLockFileMode()); diff --git a/src/nix/installables.hh b/src/nix/installables.hh index a635cb96f..9388c673e 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -8,6 +8,7 @@ namespace nix { struct Value; +struct DrvInfo; class EvalState; class SourceExprCommand; @@ -89,6 +90,8 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::ResolvedFlake & resFlake); + std::tuple toDerivation(); + std::vector toDerivations() override; Value * toValue(EvalState & state) override; diff --git a/src/nix/profile.cc b/src/nix/profile.cc new file mode 100644 index 000000000..bc5c3870e --- /dev/null +++ b/src/nix/profile.cc @@ -0,0 +1,234 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "archive.hh" +#include "builtins/buildenv.hh" +#include "flake/flakeref.hh" + +#include + +using namespace nix; + +struct ProfileElementSource +{ + FlakeRef originalRef; + FlakeRef resolvedRef; + std::string attrPath; + // FIXME: output names +}; + +struct ProfileElement +{ + PathSet storePaths; + std::optional source; + bool active = true; + // FIXME: priority +}; + +struct ProfileManifest +{ + std::vector elements; + + ProfileManifest(const Path & profile) + { + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) + element.storePaths.insert((std::string) p); + element.active = e["active"]; + if (e.value("uri", "") != "") { + element.source = ProfileElementSource{ + FlakeRef(e["originalUri"]), + FlakeRef(e["uri"]), + e["attrPath"] + }; + } + elements.emplace_back(std::move(element)); + } + } + } + + std::string toJSON() const + { + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) + paths.push_back(path); + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + if (element.source) { + obj["originalUri"] = element.source->originalRef.to_string(); + obj["uri"] = element.source->resolvedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 1; + json["elements"] = array; + return json.dump(); + } + + Path build(ref store) + { + auto tempDir = createTempDir(); + + ValidPathInfo info; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) + pkgs.emplace_back(path, true, 5); + info.references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON()); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.path = store->makeFixedOutputPath(true, info.narHash, "profile", info.references); + info.ca = makeFixedOutputCA(true, info.narHash); + + store->addToStore(info, sink.s); + + return info.path; + } +}; + +struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile +{ + std::string description() override + { + return "install a package into a profile"; + } + + Examples examples() override + { + return { + Example{ + "To install a package from Nixpkgs:", + "nix profile install nixpkgs#hello" + }, + Example{ + "To install a package from a specific branch of Nixpkgs:", + "nix profile install nixpkgs/release-19.09#hello" + }, + Example{ + "To install a package from a specific revision of Nixpkgs:", + "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*profile); + + PathSet pathsToBuild; + + for (auto & installable : installables) { + if (auto installable2 = std::dynamic_pointer_cast(installable)) { + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + + ProfileElement element; + element.storePaths = {drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable2->flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.insert(makeDrvPathWithOutputs(drv.drvPath, {"out"})); // FIXME + + manifest.elements.emplace_back(std::move(element)); + } else + throw Error("'nix profile install' does not support argument '%s'", installable->what()); + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "info"; + } + + Examples examples() override + { + return { + Example{ + "To show what packages are installed in the default profile:", + "nix profile info" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*profile); + + for (auto & element : manifest.elements) { + std::cout << fmt("%s %s\n", + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + concatStringsSep(" ", element.storePaths)); + } + } +}; + +struct CmdProfile : virtual MultiCommand, virtual Command +{ + CmdProfile() + : MultiCommand({ + {"install", []() { return make_ref(); }}, + {"info", []() { return make_ref(); }}, + }) + { } + + std::string description() override + { + return "manage Nix profiles"; + } + + void run() override + { + if (!command) + throw UsageError("'nix profile' requires a sub-command."); + command->prepare(); + command->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand("profile"); + From 555ca59f2b34bb8f3e738789e9548895766609cf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 00:28:16 +0200 Subject: [PATCH 0588/2548] nix profile info: Index elements --- src/nix/profile.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index bc5c3870e..2303900c0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -14,6 +14,7 @@ using namespace nix; struct ProfileElementSource { FlakeRef originalRef; + // FIXME: record original attrpath. FlakeRef resolvedRef; std::string attrPath; // FIXME: output names @@ -193,8 +194,9 @@ struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile { ProfileManifest manifest(*profile); - for (auto & element : manifest.elements) { - std::cout << fmt("%s %s\n", + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + std::cout << fmt("%d %s %s\n", i, element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", concatStringsSep(" ", element.storePaths)); From e30a0155d47a2e8a2eb9d0801b8b1602f71c5fd7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 13:06:32 +0200 Subject: [PATCH 0589/2548] Add "nix profile remove" command --- src/libstore/builtins/buildenv.cc | 2 +- src/nix/profile.cc | 109 +++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c1c85d0bf..1b802d908 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -156,7 +156,7 @@ void buildProfile(const Path & out, Packages && pkgs) addPkg(pkgDir, priorityCounter++); } - printError("created %d symlinks in user environment", state.symlinks); + debug("created %d symlinks in user environment", state.symlinks); } void builtinBuildenv(const BasicDerivation & drv) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 2303900c0..c9a6a5355 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include +#include using namespace nix; @@ -32,6 +33,8 @@ struct ProfileManifest { std::vector elements; + ProfileManifest() { } + ProfileManifest(const Path & profile) { auto manifestPath = profile + "/manifest.json"; @@ -173,11 +176,112 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } }; +class MixProfileElementMatchers : virtual Args +{ + std::vector _matchers; + +public: + + MixProfileElementMatchers() + { + expectArgs("elements", &_matchers); + } + + typedef std::variant Matcher; + + std::vector getMatchers(ref store) + { + std::vector res; + + for (auto & s : _matchers) { + size_t n; + if (string2Int(s, n)) + res.push_back(n); + else if (store->isStorePath(s)) + res.push_back(s); + else + res.push_back(std::regex(s, std::regex::extended | std::regex::icase)); + } + + return res; + } + + bool matches(const ProfileElement & element, size_t pos, std::vector matchers) + { + for (auto & matcher : matchers) { + if (auto n = std::get_if(&matcher)) { + if (*n == pos) return true; + } else if (auto path = std::get_if(&matcher)) { + if (element.storePaths.count(*path)) return true; + } else if (auto regex = std::get_if(&matcher)) { + if (element.source + && std::regex_match(element.source->attrPath, *regex)) + return true; + } + } + + return false; + } +}; + +struct CmdProfileRemove : virtual StoreCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "remove packages from a profile"; + } + + Examples examples() override + { + return { + Example{ + "To remove a package by attribute path:", + "nix profile remove packages.x86_64-linux.hello" + }, + Example{ + "To remove all package:", + "nix profile remove '.*'" + }, + Example{ + "To remove a package by store path:", + "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" + }, + Example{ + "To remove a package by position:", + "nix profile remove 3" + }, + }; + } + + void run(ref store) override + { + ProfileManifest oldManifest(*profile); + + auto matchers = getMatchers(store); + + ProfileManifest newManifest; + + for (size_t i = 0; i < oldManifest.elements.size(); ++i) { + auto & element(oldManifest.elements[i]); + if (!matches(element, i, matchers)) + newManifest.elements.push_back(element); + } + + // FIXME: warn about unused matchers? + + printInfo("removed %d packages, kept %d packages", + oldManifest.elements.size() - newManifest.elements.size(), + newManifest.elements.size()); + + updateProfile(newManifest.build(store)); + } +}; + struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile { std::string description() override { - return "info"; + return "list installed packages"; } Examples examples() override @@ -196,7 +300,7 @@ struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); - std::cout << fmt("%d %s %s\n", i, + std::cout << fmt("%d %s %s %s\n", i, element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", concatStringsSep(" ", element.storePaths)); @@ -209,6 +313,7 @@ struct CmdProfile : virtual MultiCommand, virtual Command CmdProfile() : MultiCommand({ {"install", []() { return make_ref(); }}, + {"remove", []() { return make_ref(); }}, {"info", []() { return make_ref(); }}, }) { } From af786432c53a0eb4f7a0aea2d0faf5f0655cbb05 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 14:44:51 +0200 Subject: [PATCH 0590/2548] Add "nix profile upgrade" command --- src/nix/profile.cc | 68 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c9a6a5355..8d387ef2e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -239,7 +239,7 @@ struct CmdProfileRemove : virtual StoreCommand, MixDefaultProfile, MixProfileEle "nix profile remove packages.x86_64-linux.hello" }, Example{ - "To remove all package:", + "To remove all packages:", "nix profile remove '.*'" }, Example{ @@ -277,6 +277,71 @@ struct CmdProfileRemove : virtual StoreCommand, MixDefaultProfile, MixProfileEle } }; +struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "upgrade packages using their most recent flake"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade all packages that were installed using a mutable flake reference:", + "nix profile upgrade '.*'" + }, + Example{ + "To upgrade a specific package:", + "nix profile upgrade packages.x86_64-linux.hello" + }, + }; + } + + void run(ref store) override + { + ProfileManifest manifest(*profile); + + auto matchers = getMatchers(store); + + // FIXME: code duplication + PathSet pathsToBuild; + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (element.source + && !element.source->originalRef.isImmutable() + && matches(element, i, matchers)) + { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + InstallableFlake installable(*this, FlakeRef(element.source->originalRef), {element.source->attrPath}); + + auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + + if (element.source->resolvedRef == resolvedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->resolvedRef, resolvedRef); + + element.storePaths = {drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable.flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.insert(makeDrvPathWithOutputs(drv.drvPath, {"out"})); // FIXME + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile { std::string description() override @@ -314,6 +379,7 @@ struct CmdProfile : virtual MultiCommand, virtual Command : MultiCommand({ {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, + {"upgrade", []() { return make_ref(); }}, {"info", []() { return make_ref(); }}, }) { } From ac9b427541cdc36fb696d6fe413872e12151ec3d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 15:16:57 +0200 Subject: [PATCH 0591/2548] Convert old-style profile manifest --- src/nix/profile.cc | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8d387ef2e..786ebddef 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "builtins/buildenv.hh" #include "flake/flakeref.hh" +#include "nix-env/user-env.hh" #include #include @@ -35,7 +36,7 @@ struct ProfileManifest ProfileManifest() { } - ProfileManifest(const Path & profile) + ProfileManifest(EvalState & state, const Path & profile) { auto manifestPath = profile + "/manifest.json"; @@ -61,6 +62,22 @@ struct ProfileManifest elements.emplace_back(std::move(element)); } } + + else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->followLinksToStore(profile)); + state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); + } + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {drvInfo.queryOutPath()}; + elements.emplace_back(std::move(element)); + } + } } std::string toJSON() const @@ -147,7 +164,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile void run(ref store) override { - ProfileManifest manifest(*profile); + ProfileManifest manifest(*getEvalState(), *profile); PathSet pathsToBuild; @@ -224,7 +241,7 @@ public: } }; -struct CmdProfileRemove : virtual StoreCommand, MixDefaultProfile, MixProfileElementMatchers +struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers { std::string description() override { @@ -255,7 +272,7 @@ struct CmdProfileRemove : virtual StoreCommand, MixDefaultProfile, MixProfileEle void run(ref store) override { - ProfileManifest oldManifest(*profile); + ProfileManifest oldManifest(*getEvalState(), *profile); auto matchers = getMatchers(store); @@ -300,7 +317,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf void run(ref store) override { - ProfileManifest manifest(*profile); + ProfileManifest manifest(*getEvalState(), *profile); auto matchers = getMatchers(store); @@ -342,7 +359,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile +struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { @@ -361,7 +378,7 @@ struct CmdProfileInfo : virtual StoreCommand, MixDefaultProfile void run(ref store) override { - ProfileManifest manifest(*profile); + ProfileManifest manifest(*getEvalState(), *profile); for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); From 9cac895406724e0304dff140379783c4d786e855 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 20:12:23 +0200 Subject: [PATCH 0592/2548] Use upstream nlohmann_json --- flake.lock | 6 +- flake.nix | 5 +- src/nlohmann/json.hpp | 20842 ------------------------------------ src/nlohmann/json_fwd.hpp | 64 - 4 files changed, 6 insertions(+), 20911 deletions(-) delete mode 100644 src/nlohmann/json.hpp delete mode 100644 src/nlohmann/json_fwd.hpp diff --git a/flake.lock b/flake.lock index 05e3a6e25..10c9921bf 100644 --- a/flake.lock +++ b/flake.lock @@ -2,9 +2,9 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", - "originalUri": "nixpkgs/release-19.03", - "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" + "narHash": "sha256-D1YSQzR6Xo2cNZ4sf+WCak2fqE/CIbwpZbPKd2B2Syc=", + "originalUrl": "nixpkgs/release-19.09", + "url": "github:edolstra/nixpkgs/e3d44c4fc62c99b850c792429cb61b946b66a704" } }, "version": 3 diff --git a/flake.nix b/flake.nix index 2f0073cc8..a16ef4656 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.uri = "nixpkgs/release-19.03"; + inputs.nixpkgs.uri = "nixpkgs/release-19.09"; outputs = { self, nixpkgs }: @@ -73,6 +73,7 @@ bzip2 xz brotli editline openssl pkgconfig sqlite boehmgc boost + nlohmann_json # Tests git @@ -313,7 +314,7 @@ doInstallCheck = true; - lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ]; + lcovFilter = [ "*/boost/*" "*-tab.*" "*/linenoise/*" ]; # We call `dot', and even though we just use it to # syntax-check generated dot files, it still requires some diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp deleted file mode 100644 index 5003a4fa2..000000000 --- a/src/nlohmann/json.hpp +++ /dev/null @@ -1,20842 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.6.1 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2019 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#ifndef INCLUDE_NLOHMANN_JSON_HPP_ -#define INCLUDE_NLOHMANN_JSON_HPP_ - -#define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 6 -#define NLOHMANN_JSON_VERSION_PATCH 1 - -#include // all_of, find, for_each -#include // assert -#include // and, not, or -#include // nullptr_t, ptrdiff_t, size_t -#include // hash, less -#include // initializer_list -#include // istream, ostream -#include // random_access_iterator_tag -#include // unique_ptr -#include // accumulate -#include // string, stoi, to_string -#include // declval, forward, move, pair, swap -#include // vector - -// #include - - -#include - -// #include - - -#include // transform -#include // array -#include // and, not -#include // forward_list -#include // inserter, front_inserter, end -#include // map -#include // string -#include // tuple, make_tuple -#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible -#include // unordered_map -#include // pair, declval -#include // valarray - -// #include - - -#include // exception -#include // runtime_error -#include // to_string - -// #include - - -#include // size_t - -namespace nlohmann -{ -namespace detail -{ -/// struct to capture the start position of the current token -struct position_t -{ - /// the total number of characters read - std::size_t chars_read_total = 0; - /// the number of characters read in the current line - std::size_t chars_read_current_line = 0; - /// the number of lines read - std::size_t lines_read = 0; - - /// conversion to size_t to preserve SAX interface - constexpr operator size_t() const - { - return chars_read_total; - } -}; - -} // namespace detail -} // namespace nlohmann - - -namespace nlohmann -{ -namespace detail -{ -//////////////// -// exceptions // -//////////////// - -/*! -@brief general exception of the @ref basic_json class - -This class is an extension of `std::exception` objects with a member @a id for -exception ids. It is used as the base class for all exceptions thrown by the -@ref basic_json class. This class can hence be used as "wildcard" to catch -exceptions. - -Subclasses: -- @ref parse_error for exceptions indicating a parse error -- @ref invalid_iterator for exceptions indicating errors with iterators -- @ref type_error for exceptions indicating executing a member function with - a wrong type -- @ref out_of_range for exceptions indicating access out of the defined range -- @ref other_error for exceptions indicating other library errors - -@internal -@note To have nothrow-copy-constructible exceptions, we internally use - `std::runtime_error` which can cope with arbitrary-length error messages. - Intermediate strings are built with static functions and then passed to - the actual constructor. -@endinternal - -@liveexample{The following code shows how arbitrary library exceptions can be -caught.,exception} - -@since version 3.0.0 -*/ -class exception : public std::exception -{ - public: - /// returns the explanatory string - const char* what() const noexcept override - { - return m.what(); - } - - /// the id of the exception - const int id; - - protected: - exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} - - static std::string name(const std::string& ename, int id_) - { - return "[json.exception." + ename + "." + std::to_string(id_) + "] "; - } - - private: - /// an exception object as storage for error messages - std::runtime_error m; -}; - -/*! -@brief exception indicating a parse error - -This exception is thrown by the library when a parse error occurs. Parse errors -can occur during the deserialization of JSON text, CBOR, MessagePack, as well -as when using JSON Patch. - -Member @a byte holds the byte index of the last read character in the input -file. - -Exceptions have ids 1xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. -json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. -json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. -json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. -json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. -json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. -json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. -json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. -json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. -json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. -json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. -json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. -json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). - -@note For an input with n bytes, 1 is the index of the first character and n+1 - is the index of the terminating null byte or the end of file. This also - holds true when reading a byte vector (CBOR or MessagePack). - -@liveexample{The following code shows how a `parse_error` exception can be -caught.,parse_error} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class parse_error : public exception -{ - public: - /*! - @brief create a parse error exception - @param[in] id_ the id of the exception - @param[in] pos the position where the error occurred (or with - chars_read_total=0 if the position cannot be - determined) - @param[in] what_arg the explanatory string - @return parse_error object - */ - static parse_error create(int id_, const position_t& pos, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - position_string(pos) + ": " + what_arg; - return parse_error(id_, pos.chars_read_total, w.c_str()); - } - - static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) - { - std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + - ": " + what_arg; - return parse_error(id_, byte_, w.c_str()); - } - - /*! - @brief byte index of the parse error - - The byte index of the last read character in the input file. - - @note For an input with n bytes, 1 is the index of the first character and - n+1 is the index of the terminating null byte or the end of file. - This also holds true when reading a byte vector (CBOR or MessagePack). - */ - const std::size_t byte; - - private: - parse_error(int id_, std::size_t byte_, const char* what_arg) - : exception(id_, what_arg), byte(byte_) {} - - static std::string position_string(const position_t& pos) - { - return " at line " + std::to_string(pos.lines_read + 1) + - ", column " + std::to_string(pos.chars_read_current_line); - } -}; - -/*! -@brief exception indicating errors with iterators - -This exception is thrown if iterators passed to a library function do not match -the expected semantics. - -Exceptions have ids 2xx. - -name / id | example message | description ------------------------------------ | --------------- | ------------------------- -json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. -json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. -json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. -json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. -json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. -json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. -json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. -json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. -json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. -json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. -json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. -json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). - -@liveexample{The following code shows how an `invalid_iterator` exception can be -caught.,invalid_iterator} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class invalid_iterator : public exception -{ - public: - static invalid_iterator create(int id_, const std::string& what_arg) - { - std::string w = exception::name("invalid_iterator", id_) + what_arg; - return invalid_iterator(id_, w.c_str()); - } - - private: - invalid_iterator(int id_, const char* what_arg) - : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating executing a member function with a wrong type - -This exception is thrown in case of a type error; that is, a library function is -executed on a JSON value whose type does not match the expected semantics. - -Exceptions have ids 3xx. - -name / id | example message | description ------------------------------ | --------------- | ------------------------- -json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. -json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. -json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t &. -json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. -json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. -json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. -json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. -json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. -json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. -json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. -json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. -json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. -json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. -json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. -json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. -json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | -json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | - -@liveexample{The following code shows how a `type_error` exception can be -caught.,type_error} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref out_of_range for exceptions indicating access out of the defined range -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class type_error : public exception -{ - public: - static type_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("type_error", id_) + what_arg; - return type_error(id_, w.c_str()); - } - - private: - type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating access out of the defined range - -This exception is thrown in case a library function is called on an input -parameter that exceeds the expected range, for instance in case of array -indices or nonexisting object keys. - -Exceptions have ids 4xx. - -name / id | example message | description -------------------------------- | --------------- | ------------------------- -json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. -json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. -json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. -json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. -json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. -json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | -json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | -json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | - -@liveexample{The following code shows how an `out_of_range` exception can be -caught.,out_of_range} - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref other_error for exceptions indicating other library errors - -@since version 3.0.0 -*/ -class out_of_range : public exception -{ - public: - static out_of_range create(int id_, const std::string& what_arg) - { - std::string w = exception::name("out_of_range", id_) + what_arg; - return out_of_range(id_, w.c_str()); - } - - private: - out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; - -/*! -@brief exception indicating other library errors - -This exception is thrown in case of errors that cannot be classified with the -other exception types. - -Exceptions have ids 5xx. - -name / id | example message | description ------------------------------- | --------------- | ------------------------- -json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. - -@sa - @ref exception for the base class of the library exceptions -@sa - @ref parse_error for exceptions indicating a parse error -@sa - @ref invalid_iterator for exceptions indicating errors with iterators -@sa - @ref type_error for exceptions indicating executing a member function with - a wrong type -@sa - @ref out_of_range for exceptions indicating access out of the defined range - -@liveexample{The following code shows how an `other_error` exception can be -caught.,other_error} - -@since version 3.0.0 -*/ -class other_error : public exception -{ - public: - static other_error create(int id_, const std::string& what_arg) - { - std::string w = exception::name("other_error", id_) + what_arg; - return other_error(id_, w.c_str()); - } - - private: - other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} -}; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // pair - -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them - -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #endif -#endif - -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" -#endif - -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" -#endif - -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) -#else - #define JSON_DEPRECATED -#endif - -// allow for portable nodiscard warnings -#if defined(__has_cpp_attribute) - #if __has_cpp_attribute(nodiscard) - #define JSON_NODISCARD [[nodiscard]] - #elif __has_cpp_attribute(gnu::warn_unused_result) - #define JSON_NODISCARD [[gnu::warn_unused_result]] - #else - #define JSON_NODISCARD - #endif -#else - #define JSON_NODISCARD -#endif - -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) -#else - #include - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) -#endif - -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER -#endif - -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(x, 1) - #define JSON_UNLIKELY(x) __builtin_expect(x, 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif - -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif - -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } - -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. - -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> - -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json - -// #include - - -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type - -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; - -template -using uncvref_t = typename std::remove_cv::type>::type; - -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; - -template -struct merge_and_renumber; - -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; - -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; - -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; - -template -using index_sequence_for = make_index_sequence; - -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; - -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; - -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann - -// #include - - -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval - -// #include - - -#include // random_access_iterator_tag - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann - -// #include - - -namespace nlohmann -{ -namespace detail -{ -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; - -template -struct iterator_traits::value>> -{ - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; -}; -} // namespace detail -} // namespace nlohmann - -// #include - -// #include - -// #include - - -#include - -// #include - - -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ -namespace detail -{ -struct nonesuch -{ - nonesuch() = delete; - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; - nonesuch(nonesuch const&&) = delete; - void operator=(nonesuch const&) = delete; - void operator=(nonesuch&&) = delete; -}; - -template class Op, - class... Args> -struct detector -{ - using value_t = std::false_type; - using type = Default; -}; - -template class Op, class... Args> -struct detector>, Op, Args...> -{ - using value_t = std::true_type; - using type = Op; -}; - -template