diff --git a/.gitignore b/.gitignore index 969194650..93a9ff9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ perl/Makefile.config # /tests/lang/ /tests/lang/*.out /tests/lang/*.out.xml +/tests/lang/*.err /tests/lang/*.ast /perl/lib/Nix/Config.pm diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..a5253997d 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -86,6 +86,31 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Characterization testing + +Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. +(At least, this is the convention we've used for `tests/lang.sh`. +If we add more characterization testing we should always strive to be consistent.) + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 35e3dccc3..7b2321466 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -3,7 +3,7 @@ This section lists the functions built into the Nix language evaluator. All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. -For convenience, some built-ins are can be accessed directly: +For convenience, some built-ins can be accessed directly: - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8479b166a..139d07188 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,5 +2,7 @@ - [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + - Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index f46413630..45fbea02c 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 +TasksMax=1048576 [Install] WantedBy=multi-user.target diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7f97364a1..3df2c71a5 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -105,7 +105,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20..6359b2579 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -3,6 +3,7 @@ #include "args.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional evalStoreUrl; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4b160a100..f9e9c2bf8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -68,7 +68,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store,ref state, + NixRepl(const SearchPath & searchPath, nix::ref store,ref state, std::function getValues); virtual ~NixRepl(); @@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr AbstractNixRepl::create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( @@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db..6d88883fe 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); static void runSimple( diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 706a19024..be1bdb806 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -498,7 +498,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore) : sWith(symbols.create("")) @@ -563,30 +563,32 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + addToSearchPath(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + addToSearchPath(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); + if (!r) continue; - auto path = r.second; + auto path = *std::move(r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3676c1b7..277e77ad5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include #include @@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -struct SearchPathElem -{ - std::string prefix; - // FIXME: maybe change this to an std::variant. - std::string path; -}; -typedef std::list SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -317,7 +309,7 @@ private: SearchPath searchPath; - std::map> searchPathResolved; + std::map> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -344,12 +336,12 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + void addToSearchPath(SearchPath::Elem && elem); SearchPath getSearchPath() { return searchPath; } @@ -431,12 +423,16 @@ public: * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optinal key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + std::optional resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0d0004f9f..0a1ad9967 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -663,7 +663,7 @@ Expr * EvalState::parse( ParseData data { .state = *this, .symbols = symbols, - .basePath = std::move(basePath), + .basePath = basePath, .origin = {origin}, }; @@ -734,22 +734,9 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) +void EvalState::addToSearchPath(SearchPath::Elem && elem) { - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, - }); + searchPath.elements.emplace_back(std::move(elem)); } @@ -759,22 +746,19 @@ SourcePath EvalState::findFile(const std::string_view path) } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.prefix.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.prefix.size(); - if (path.compare(0, s, i.prefix) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -791,49 +775,53 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { - auto i = searchPathResolved.find(elem.path); + auto & value = value0.s; + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair res; + std::optional res; - if (EvalSettings::isPseudoUrl(elem.path)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.path, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.path); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.path); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.path, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.path] = res; + searchPathResolved[value] = res; return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5dfad470a..8a61e57cc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1502,6 +1502,8 @@ static RegisterPrimOp primop_storePath({ in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", .fun = prim_storePath, }); @@ -1656,9 +1658,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, }); } @@ -4319,12 +4321,12 @@ void EvalState::createBaseEnv() }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.path); - attrs.alloc("prefix").mkString(i.prefix); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v, { diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index bae849f61..7fe8203f4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -5,37 +5,150 @@ namespace nix { +/** + * Handler for the content addressed case. + * + * @param state Evaluator state and store to write to. + * @param fromStore Store containing the path to rewrite. + * @param fromPath Source path to be rewritten. + * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path. + * @param v Return `Value` + */ +static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional & toPathMaybe, Value &v) { + + // establish toPath or throw + + if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) { + auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); + if (toPathMaybe && *toPathMaybe != rewrittenPath) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath), + state.store->printStorePath(*toPathMaybe)), + .errPos = state.positions[pos] + }); + if (!toPathMaybe) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'\n" + "Use this value for the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath)), + .errPos = state.positions[pos] + }); + } + + auto toPath = *toPathMaybe; + + // check and return + + auto resultInfo = state.store->queryPathInfo(toPath); + + if (!resultInfo->isContentAddressed(*state.store)) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + throw Error({ + .msg = hintfmt( + "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" + "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", + state.store->printStorePath(toPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(toPath, v); +} + +/** + * Fetch the closure and make sure it's content addressed. + */ +static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (!info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" + "If you do intend to fetch an input-addressed store path, add\n\n" + " inputAddressed = true;\n\n" + "to the 'fetchClosure' arguments.\n\n" + "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +/** + * Fetch the closure and make sure it's input addressed. + */ +static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" + "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +typedef std::optional StorePathOrGap; + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional fromStoreUrl; std::optional fromPath; - bool toCA = false; - std::optional toPath; + std::optional toPath; + std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; + auto attrHint = [&]() -> std::string { + return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + }; if (attrName == "fromPath") { NixStringContext context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - toCA = true; - if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + if (isEmptyString) { + toPath = StorePathOrGap {}; + } + else { NixStringContext context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } } else if (attrName == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, - "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); + attrHint()); + + else if (attrName == "inputAddressed") + inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint()); else throw Error({ @@ -50,6 +163,18 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg .errPos = state.positions[pos] }); + bool inputAddressed = inputAddressedMaybe.value_or(false); + + if (inputAddressed) { + if (toPath) + throw Error({ + .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + "inputAddressed", + "toPath"), + .errPos = state.positions[pos] + }); + } + if (!fromStoreUrl) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), @@ -74,55 +199,40 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (toCA) { - if (!toPath || !state.store->isValidPath(*toPath)) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), - .errPos = state.positions[pos] - }); - } - } else { - if (!state.store->isValidPath(*fromPath)) - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); - toPath = fromPath; - } - - /* In pure mode, require a CA path. */ - if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*toPath); - if (!info->isContentAddressed(*state.store)) - throw Error({ - .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } - - state.mkStorePathString(*toPath, v); + if (toPath) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); + else if (inputAddressed) + runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); + else + runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v); } static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache, rewriting it into - content-addressed form. For example, + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context. + + This function can be invoked in three ways, that we will discuss in order of preference. + + **Fetch a content-addressed store path** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. + + If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations. + + **Fetch any store path and rewrite it to a fully content-addressed store path** + + Example: ```nix builtins.fetchClosure { @@ -132,28 +242,42 @@ static RegisterPrimOp primop_fetchClosure({ } ``` - fetches `/nix/store/r2jd...` from the specified binary cache, + This example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - If `fromPath` is already content-addressed, or if you are - allowing impure evaluation (`--impure`), then `toPath` may be - omitted. + Like the previous example, no extra configuration or privileges are required. To find out the correct value for `toPath` given a `fromPath`, - you can use `nix store make-content-addressed`: + use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - This function is similar to `builtins.storePath` in that it - allows you to use a previously built store path in a Nix - expression. However, it is more reproducible because it requires - specifying a binary cache from which the path can be fetched. - Also, requiring a content-addressed final store path avoids the - need for users to configure binary cache public keys. + Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. + + **Fetch an input-addressed store path as is** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is. + However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration. + + **`builtins.storePath`** + + `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. + However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. + Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..36bb4c3a5 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,56 @@ +#include "search-path.hh" +#include "util.hh" + +namespace nix { + +std::optional SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && (path.size() - n) > 0; + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..ce78135b5 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000..dbe7ab95f --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 53e6998e8..068b47f93 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -909,9 +909,12 @@ void LocalDerivationGoal::startBuilder() /* Drop additional groups here because we can't do it after we've created the new user namespace. */ - if (settings.dropSupplementaryGroups) - if (setgroups(0, 0) == -1) - throw SysError("setgroups failed. Set the drop-supplementary-groups option to false to skip this step."); + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a19b43086..d4b8fb1f9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -524,23 +524,22 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting dropSupplementaryGroups{this, getuid() == 0, "drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", R"( - Whether to drop supplementary groups when building with sandboxing. - This is normally a good idea if we are root and have the capability to - do so. + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. - But if this "root" is mapped from a non-root user in a larger - namespace, we won't be able drop additional groups; they will be - mapped to nogroup in the child namespace. There does not seem to be a - workaround for this. + However this can fail under some circumstances. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. - (But who can tell from reading user_namespaces(7)? See also https://lwn.net/Articles/621612/.) + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. - TODO: It might be good to create a middle ground option that allows - `setgroups` to fail if all additional groups are "nogroup" / the value - of `/proc/sys/fs/overflowuid`. This would handle the common - nested-sandboxing case identified above. + This option defaults to `true` when the user is root + (since `root` usually has permissions to call setgroups) + and `false` otherwise. )"}; #if __linux__ diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704..626a22480 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -80,4 +80,15 @@ std::map makeContentAddressed( return remappings; } +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & fromPath) +{ + auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath }); + auto i = remappings.find(fromPath); + assert(i != remappings.end()); + return i->second; +} + } diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index 2ce6ec7bc..60bb2b477 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -5,9 +5,20 @@ namespace nix { +/** Rewrite a closure of store paths to be completely content addressed. + */ std::map makeContentAddressed( Store & srcStore, Store & dstStore, - const StorePathSet & storePaths); + const StorePathSet & rootPaths); + +/** Rewrite a closure of a store path to be completely content addressed. + * + * This is a convenience function for the case where you only have one root path. + */ +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & rootPath); } diff --git a/src/nix/nix.md b/src/nix/nix.md index 6d9e40dbc..e0f459d6b 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -63,7 +63,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path -For most commands, if no installable is specified, `.` as assumed. +For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. ### Flake output attribute diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bf..d05c23fb7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(storePathsUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 78e12b477..d2a2132c1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,6 +1,7 @@ requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/dependencies.sh b/tests/dependencies.sh index f9da0c6bc..d5cd30396 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -15,6 +15,9 @@ if test -n "$dot"; then $dot < $TEST_ROOT/graph fi +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index 21650eb06..a02d1ce7a 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -33,20 +33,43 @@ clearStore [ ! -e $nonCaPath ] [ -e $caPath ] +clearStore + +# The daemon will reject input addressed paths unless configured to trust the +# cache key or the user. This behavior should be covered by another test, so we +# skip this part when using the daemon. if [[ "$NIX_REMOTE" != "daemon" ]]; then - # In impure mode, we can use non-CA paths. - [[ $(nix eval --raw --no-require-sigs --impure --expr " + # If we want to return a non-CA path, we have to be explicit about it. + expectStderr 1 nix eval --raw --no-require-sigs --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; fromPath = $nonCaPath; } + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." + + # TODO: Should the closure be rejected, despite single user mode? + # [ ! -e $nonCaPath ] + + [ ! -e $caPath ] + + # We can use non-CA paths when we ask explicitly. + [[ $(nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + inputAddressed = true; + } ") = $nonCaPath ]] [ -e $nonCaPath ] + [ ! -e $caPath ] + fi +[ ! -e $caPath ] + # 'toPath' set to empty string should fail but print the expected path. expectStderr 1 nix eval -v --json --expr " builtins.fetchClosure { @@ -59,6 +82,10 @@ expectStderr 1 nix eval -v --json --expr " # If fromPath is CA, then toPath isn't needed. nix copy --to file://$cacheDir $caPath +clearStore + +[ ! -e $caPath ] + [[ $(nix eval -v --raw --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; @@ -66,6 +93,8 @@ nix copy --to file://$cacheDir $caPath } ") = $caPath ]] +[ -e $caPath ] + # Check that URL query parameters aren't allowed. clearStore narCache=$TEST_ROOT/nar-cache @@ -77,3 +106,45 @@ rm -rf $narCache } ") (! [ -e $narCache ]) + +# If toPath is specified but wrong, we check it (only) when the path is missing. +clearStore + +badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') + +[ ! -e $badPath ] + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $badPath; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" + +[ ! -e $badPath ] + +# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. +# It would be nice for this to fail, but checking it would be too(?) slow. +[ -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $badPath; + toPath = $caPath; + } +") = $caPath ]] + + +# However, if the output address is unexpected, we can report it + + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + inputAddressed = true; + } +" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*' + diff --git a/tests/lang-test-infra.sh b/tests/lang-test-infra.sh new file mode 100644 index 000000000..30da8977b --- /dev/null +++ b/tests/lang-test-infra.sh @@ -0,0 +1,86 @@ +# Test the function for lang.sh +source common.sh + +source lang/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/lang.sh b/tests/lang.sh old mode 100644 new mode 100755 index 8170cb39d..75dbbc38e --- a/tests/lang.sh +++ b/tests/lang.sh @@ -1,5 +1,17 @@ source common.sh +set -o pipefail + +source lang/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + export TEST_VAR=foo # for eval-okay-getenv.nix export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store @@ -20,63 +32,115 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t set +x -fail=0 +badDiff=0 +badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't parse" - fail=1 + badExitCode=1 fi done for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should parse" - fail=1 + badExitCode=1 fi done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then + i=$(basename "$i" .nix) + if + expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't evaluate" - fail=1 + badExitCode=1 fi done for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) + i=$(basename "$i" .nix) - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + diffAndAccept "$i" out.xml exp.xml + else echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi - fi - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 + badExitCode=1 fi fi done -exit $fail +if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + else + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi +else + if (( "$badDiff" )); then + echo '' + echo 'You can rerun this test with:' + echo '' + echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + fi + exit $(( "$badExitCode" + "$badDiff" )) +fi diff --git a/tests/lang/empty.exp b/tests/lang/empty.exp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000..345232d3f --- /dev/null +++ b/tests/lang/eval-fail-abort.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-abort.nix:1:14: + + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/lang/eval-fail-antiquoted-path.err.exp new file mode 100644 index 000000000..425deba42 --- /dev/null +++ b/tests/lang/eval-fail-antiquoted-path.err.exp @@ -0,0 +1 @@ +error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000..aeecd8167 --- /dev/null +++ b/tests/lang/eval-fail-assert.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-assert.nix:4:3: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + + at /pwd/lang/eval-fail-assert.nix:4:10: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + + at /pwd/lang/eval-fail-assert.nix:2:7: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: assertion '(arg == "y")' failed + + at /pwd/lang/eval-fail-assert.nix:2:12: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/lang/eval-fail-bad-antiquote-1.err.exp new file mode 100644 index 000000000..cf94f53bc --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/lang/eval-fail-bad-antiquote-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/lang/eval-fail-bad-antiquote-3.err.exp new file mode 100644 index 000000000..fbefbc826 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000..eb73e9a52 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000..ac14f329b --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000..f0618d8ac --- /dev/null +++ b/tests/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-blackhole.nix:2:3: + + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-blackhole.nix:3:7: + + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000..5e204ba73 --- /dev/null +++ b/tests/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'deepSeq' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:1: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + + at /pwd/lang/eval-fail-deepseq.nix:1:20: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:24: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000..0069285fb --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,38 @@ +error: + … while calling the 'foldl'' builtin + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000..f6bd19f5a --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'fromTOML' builtin + + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing a TOML string: Dates and times are not supported + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000..8e77dec1e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,19 @@ +error: + … while calling the 'toString' builtin + + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + at «none»:0: (source not available) + + … while calling the 'hashFile' builtin + + at «none»:0: (source not available) + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-list.err.exp b/tests/lang/eval-fail-list.err.exp new file mode 100644 index 000000000..24d682118 --- /dev/null +++ b/tests/lang/eval-fail-list.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating one of the elements to concatenate + + at /pwd/lang/eval-fail-list.nix:1:2: + + 1| 8++1 + | ^ + 2| + + error: value is an integer while a list was expected diff --git a/tests/lang/eval-fail-list.nix b/tests/lang/eval-fail-list.nix new file mode 100644 index 000000000..fa749f2f7 --- /dev/null +++ b/tests/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000..61fabf0d5 --- /dev/null +++ b/tests/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,16 @@ +error: + … from call site + + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000..f0011c97f --- /dev/null +++ b/tests/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,8 @@ +error: path has a trailing slash + + at /pwd/lang/eval-fail-path-slash.nix:6:12: + + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000..af64133cb --- /dev/null +++ b/tests/lang/eval-fail-recursion.err.exp @@ -0,0 +1,16 @@ +error: + … in the right operand of the update (//) operator + + at /pwd/lang/eval-fail-recursion.nix:1:12: + + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-recursion.nix:1:15: + + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/lang/eval-fail-recursion.nix b/tests/lang/eval-fail-recursion.nix new file mode 100644 index 000000000..075b5ed06 --- /dev/null +++ b/tests/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000..e82cdac98 --- /dev/null +++ b/tests/lang/eval-fail-remove.err.exp @@ -0,0 +1,19 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-remove.nix:4:3: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + + at /pwd/lang/eval-fail-remove.nix:4:10: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000..22b6166f8 --- /dev/null +++ b/tests/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-scope-5.nix:8:3: + + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + + at /pwd/lang/eval-fail-scope-5.nix:8:10: + + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + + at /pwd/lang/eval-fail-scope-5.nix:6:7: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-scope-5.nix:6:12: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000..33a7e9491 --- /dev/null +++ b/tests/lang/eval-fail-seq.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'seq' builtin + + at /pwd/lang/eval-fail-seq.nix:1:1: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-seq.nix:1:15: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000..beb29d678 --- /dev/null +++ b/tests/lang/eval-fail-set-override.err.exp @@ -0,0 +1,6 @@ +error: + … while evaluating the `__overrides` attribute + + at «none»:0: (source not available) + + error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-set-override.nix b/tests/lang/eval-fail-set-override.nix new file mode 100644 index 000000000..03551c186 --- /dev/null +++ b/tests/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/lang/eval-fail-set.err.exp b/tests/lang/eval-fail-set.err.exp new file mode 100644 index 000000000..0d0140508 --- /dev/null +++ b/tests/lang/eval-fail-set.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'x' + + at /pwd/lang/eval-fail-set.nix:1:3: + + 1| 8.x + | ^ + 2| diff --git a/tests/lang/eval-fail-set.nix b/tests/lang/eval-fail-set.nix new file mode 100644 index 000000000..c6b7980b6 --- /dev/null +++ b/tests/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000..dc26a00bd --- /dev/null +++ b/tests/lang/eval-fail-substring.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'substring' builtin + + at /pwd/lang/eval-fail-substring.nix:1:1: + + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000..43ed2bdfc --- /dev/null +++ b/tests/lang/eval-fail-to-path.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'toPath' builtin + + at /pwd/lang/eval-fail-to-path.nix:1:1: + + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + at «none»:0: (source not available) + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000..30db743c7 --- /dev/null +++ b/tests/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,17 @@ +error: + … from call site + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix index e1c0f86cc..4c526b9ae 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-overrides.nix b/tests/lang/eval-okay-overrides.nix index 358742b36..719bdc9c0 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/lang/eval-okay-print.err.exp b/tests/lang/eval-okay-print.err.exp new file mode 100644 index 000000000..3fc99be3e --- /dev/null +++ b/tests/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ ] diff --git a/tests/lang/eval-okay-print.exp b/tests/lang/eval-okay-print.exp new file mode 100644 index 000000000..0d960fb70 --- /dev/null +++ b/tests/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/lang/eval-okay-print.nix b/tests/lang/eval-okay-print.nix new file mode 100644 index 000000000..d36ba4da3 --- /dev/null +++ b/tests/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags index a28e68210..dfad1c611 100644 --- a/tests/lang/eval-okay-search-path.flags +++ b/tests/lang/eval-okay-search-path.flags @@ -1 +1 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh new file mode 100644 index 000000000..516bff8ad --- /dev/null +++ b/tests/lang/framework.sh @@ -0,0 +1,33 @@ +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=lang/empty.exp + fi + + # Diff so we get a nice message + if ! diff --unified "$got" "$expectedOrEmpty"; then + echo "FAIL: evaluation result of $testName not as expected" + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000..4fe6b7a1f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:1:3 + + at «stdin»:3:3: + + 2| y = 456; + 3| x = 789; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000..ff68446a1 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,7 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + + at «stdin»:3:3: + + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/lang/parse-fail-dup-attrs-6.err.exp new file mode 100644 index 000000000..74823fc25 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-6.err.exp @@ -0,0 +1 @@ +error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000..512a499ca --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:6:12 + + at «stdin»:7:12: + + 6| inherit x; + 7| inherit x; + | ^ diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000..1d566fb33 --- /dev/null +++ b/tests/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,6 @@ +error: duplicate formal function argument 'x' + + at «stdin»:1:8: + + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000..f9fa72312 --- /dev/null +++ b/tests/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected end of file, expecting '"' + + at «stdin»:3:5: + + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000..32f776795 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,8 @@ +error: attribute 'z' already defined at «stdin»:3:16 + + at «stdin»:2:3: + + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000..0437cd50c --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,8 @@ +error: attribute 'y' already defined at «stdin»:3:9 + + at «stdin»:2:3: + + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000..634a04aaa --- /dev/null +++ b/tests/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,7 @@ +error: duplicate formal function argument 'args' + + at «stdin»:1:1: + + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000..167d01e85 --- /dev/null +++ b/tests/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,8 @@ +error: undefined variable 'gcc' + + at «stdin»:8:12: + + 7| + 8| body = ({ + | ^ + 9| inherit gcc; diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000..77c96bbd2 --- /dev/null +++ b/tests/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected ':', expecting '}' + + at «stdin»:3:13: + + 2| + 3| f = {x, y : + | ^ diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000..48e88747f --- /dev/null +++ b/tests/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'y' + + at «stdin»:1:4: + + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000..6087479a3 --- /dev/null +++ b/tests/lang/parse-fail-utf8.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected invalid token, expecting end of file + + at «stdin»:1:5: + + 1| 123 à + | ^ diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/lang/parse-fail-utf8.nix diff --git a/tests/lang/parse-okay-1.exp b/tests/lang/parse-okay-1.exp new file mode 100644 index 000000000..d5ab5f18a --- /dev/null +++ b/tests/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-crlf.exp b/tests/lang/parse-okay-crlf.exp new file mode 100644 index 000000000..4213609fc --- /dev/null +++ b/tests/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000..b89a59734 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000..9df7219e4 --- /dev/null +++ b/tests/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000..e2ed886fe --- /dev/null +++ b/tests/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-subversion.exp b/tests/lang/parse-okay-subversion.exp new file mode 100644 index 000000000..4168ee8bf --- /dev/null +++ b/tests/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-url.exp b/tests/lang/parse-okay-url.exp new file mode 100644 index 000000000..e5f0829b0 --- /dev/null +++ b/tests/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/local.mk b/tests/local.mk index 2be1081f7..acd9e2824 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -20,6 +20,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ diff --git a/tests/misc.sh b/tests/misc.sh index 60d58310e..af96d20bd 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" diff --git a/tests/restricted.sh b/tests/restricted.sh index 776893a56..17f310a4b 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" diff --git a/tests/signing.sh b/tests/signing.sh index 9b673c609..942b51630 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -84,6 +84,10 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) # Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 + +# Fails or very flaky only on GHA + macOS: +# expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' +# but this works: (! nix copy --to $TEST_ROOT/store0 $outPath) # But succeed if we supply the public keys. diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh index 47debc5e3..d18fb2414 100644 --- a/tests/supplementary-groups.sh +++ b/tests/supplementary-groups.sh @@ -8,6 +8,10 @@ needLocalStore "The test uses --store always so we would just be bypassing the d unshare --mount --map-root-user bash <