nix-super/src/libexpr/eval.cc

3083 lines
97 KiB
C++
Raw Normal View History

#include "eval.hh"
#include "eval-settings.hh"
#include "hash.hh"
2023-11-16 14:04:27 +02:00
#include "primops.hh"
#include "print-options.hh"
#include "exit.hh"
#include "types.hh"
#include "util.hh"
#include "store-api.hh"
#include "derivations.hh"
#include "downstream-placeholder.hh"
#include "eval-inline.hh"
#include "filetransfer.hh"
2019-11-13 18:38:37 +02:00
#include "function-trace.hh"
#include "profiles.hh"
#include "print.hh"
#include "filtering-source-accessor.hh"
#include "memory-source-accessor.hh"
#include "signals.hh"
#include "gc-small-vector.hh"
#include "url.hh"
#include "fetch-to-store.hh"
#include "tarball.hh"
#include "parser-tab.hh"
2014-03-30 01:49:23 +02:00
#include <algorithm>
#include <chrono>
2022-11-29 01:25:36 +02:00
#include <iostream>
Remove 100s of CPU time (10%) from build times (1465s -> 1302s) Result's from Mic92's framework 13th Gen Intel Core i7-1360P: Before: 3595.92s user 183.01s system 1360% cpu 4:37.74 total After: 3486.07s user 168.93s system 1354% cpu 4:29.79 total I saw that boost/lexical_cast was costing about 100s in CPU time on our compiles. We can fix this trivially by doing explicit template instantiation in exactly one place and eliminating all other includes of it, which is a code improvement anyway by hiding the boost. Before: ``` lix/lix2 » ClangBuildAnalyzer --analyze buildtimeold.bin Analyzing build trace from 'buildtimeold.bin'... **** Time summary: Compilation (551 times): Parsing (frontend): 1465.3 s Codegen & opts (backend): 1110.9 s <snip> **** Expensive headers: 178153 ms: ../src/libcmd/installable-value.hh (included 52 times, avg 3426 ms), included via: 40x: command.hh 5x: command-installable-value.hh 3x: installable-flake.hh 2x: <direct include> 2x: installable-attr-path.hh 176217 ms: ../src/libutil/error.hh (included 246 times, avg 716 ms), included via: 36x: command.hh installable-value.hh installables.hh derived-path.hh config.hh experimental-features.hh 12x: globals.hh config.hh experimental-features.hh 11x: file-system.hh file-descriptor.hh 6x: serialise.hh strings.hh 6x: <direct include> 6x: archive.hh serialise.hh strings.hh ... 173243 ms: ../src/libstore/store-api.hh (included 152 times, avg 1139 ms), included via: 55x: <direct include> 39x: command.hh installable-value.hh installables.hh 7x: libexpr.hh 4x: local-store.hh 4x: command-installable-value.hh installable-value.hh installables.hh 3x: binary-cache-store.hh ... 170482 ms: ../src/libutil/serialise.hh (included 201 times, avg 848 ms), included via: 37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh 14x: store-api.hh nar-info.hh hash.hh 11x: <direct include> 7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh 7x: libexpr.hh value.hh source-path.hh archive.hh 6x: fetchers.hh hash.hh ... 169397 ms: ../src/libcmd/installables.hh (included 53 times, avg 3196 ms), included via: 40x: command.hh installable-value.hh 5x: command-installable-value.hh installable-value.hh 3x: installable-flake.hh installable-value.hh 2x: <direct include> 1x: installable-derived-path.hh 1x: installable-value.hh ... 159740 ms: ../src/libutil/strings.hh (included 221 times, avg 722 ms), included via: 37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh serialise.hh 19x: <direct include> 14x: store-api.hh nar-info.hh hash.hh serialise.hh 11x: serialise.hh 7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh 7x: libexpr.hh value.hh source-path.hh archive.hh serialise.hh ... 156796 ms: ../src/libcmd/command.hh (included 51 times, avg 3074 ms), included via: 42x: <direct include> 7x: command-installable-value.hh 2x: installable-attr-path.hh 150392 ms: ../src/libutil/types.hh (included 251 times, avg 599 ms), included via: 36x: command.hh installable-value.hh installables.hh path.hh 11x: file-system.hh 10x: globals.hh 6x: fetchers.hh 6x: serialise.hh strings.hh error.hh 5x: archive.hh ... 133101 ms: /nix/store/644b90j1vms44nr18yw3520pzkrg4dd1-boost-1.81.0-dev/include/boost/lexical_cast.hpp (included 226 times, avg 588 ms), included via : 37x: command.hh installable-value.hh installables.hh built-path.hh realisation.hh hash.hh serialise.hh strings.hh 19x: file-system.hh 11x: store-api.hh nar-info.hh hash.hh serialise.hh strings.hh 7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh 7x: libexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh 6x: eval.hh attr-set.hh nixexpr.hh value.hh source-path.hh archive.hh serialise.hh strings.hh ... 132887 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/memory (included 262 times, avg 507 ms), included via: 36x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh 16x: gtest.h 11x: file-system.hh types.hh ref.hh 10x: globals.hh types.hh ref.hh 10x: json.hpp 6x: serialise.hh ... done in 0.6s. ``` After: ``` lix/lix2 » maintainers/buildtime_report.sh build Processing all files and saving to '/home/jade/lix/lix2/maintainers/../buildtime.bin'... done in 0.6s. Run 'ClangBuildAnalyzer --analyze /home/jade/lix/lix2/maintainers/../buildtime.bin' to analyze it. Analyzing build trace from '/home/jade/lix/lix2/maintainers/../buildtime.bin'... **** Time summary: Compilation (551 times): Parsing (frontend): 1302.1 s Codegen & opts (backend): 956.3 s <snip> **** Expensive headers: 178145 ms: ../src/libutil/error.hh (included 246 times, avg 724 ms), included via: 36x: command.hh installable-value.hh installables.hh derived-path.hh config.hh experimental-features.hh 12x: globals.hh config.hh experimental-features.hh 11x: file-system.hh file-descriptor.hh 6x: <direct include> 6x: serialise.hh strings.hh 6x: fetchers.hh hash.hh serialise.hh strings.hh ... 154043 ms: ../src/libcmd/installable-value.hh (included 52 times, avg 2962 ms), included via: 40x: command.hh 5x: command-installable-value.hh 3x: installable-flake.hh 2x: <direct include> 2x: installable-attr-path.hh 153593 ms: ../src/libstore/store-api.hh (included 152 times, avg 1010 ms), included via: 55x: <direct include> 39x: command.hh installable-value.hh installables.hh 7x: libexpr.hh 4x: local-store.hh 4x: command-installable-value.hh installable-value.hh installables.hh 3x: binary-cache-store.hh ... 149948 ms: ../src/libutil/types.hh (included 251 times, avg 597 ms), included via: 36x: command.hh installable-value.hh installables.hh path.hh 11x: file-system.hh 10x: globals.hh 6x: fetchers.hh 6x: serialise.hh strings.hh error.hh 5x: archive.hh ... 144560 ms: ../src/libcmd/installables.hh (included 53 times, avg 2727 ms), included via: 40x: command.hh installable-value.hh 5x: command-installable-value.hh installable-value.hh 3x: installable-flake.hh installable-value.hh 2x: <direct include> 1x: installable-value.hh 1x: installable-derived-path.hh ... 136585 ms: ../src/libcmd/command.hh (included 51 times, avg 2678 ms), included via: 42x: <direct include> 7x: command-installable-value.hh 2x: installable-attr-path.hh 133394 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/memory (included 262 times, avg 509 ms), included via: 36x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh 16x: gtest.h 11x: file-system.hh types.hh ref.hh 10x: globals.hh types.hh ref.hh 10x: json.hpp 6x: serialise.hh ... 89315 ms: ../src/libstore/derived-path.hh (included 178 times, avg 501 ms), included via: 37x: command.hh installable-value.hh installables.hh 25x: store-api.hh realisation.hh 7x: primops.hh eval.hh attr-set.hh nixexpr.hh value.hh context.hh 6x: eval.hh attr-set.hh nixexpr.hh value.hh context.hh 6x: libexpr.hh value.hh context.hh 6x: shared.hh ... 87347 ms: /nix/store/h2abv2l8irqj942i5rq9wbrj42kbsh5y-gcc-12.3.0/include/c++/12.3.0/ostream (included 273 times, avg 319 ms), included via: 35x: command.hh installable-value.hh installables.hh path.hh types.hh ref.hh memory unique_ptr.h 12x: regex sstream istream 10x: file-system.hh types.hh ref.hh memory unique_ptr.h 10x: gtest.h memory unique_ptr.h 10x: globals.hh types.hh ref.hh memory unique_ptr.h 6x: fetchers.hh types.hh ref.hh memory unique_ptr.h ... 85249 ms: ../src/libutil/config.hh (included 213 times, avg 400 ms), included via: 37x: command.hh installable-value.hh installables.hh derived-path.hh 20x: globals.hh 20x: logging.hh 16x: store-api.hh logging.hh 6x: <direct include> 6x: eval.hh attr-set.hh nixexpr.hh value.hh context.hh derived-path.hh ... done in 0.5s. ``` Adapated from https://git.lix.systems/lix-project/lix/commit/18aa3e1d570b4ecbb9962376e5fba5757dad8da9
2024-05-30 07:12:34 +03:00
#include <sstream>
#include <cstring>
2022-11-29 01:25:36 +02:00
#include <optional>
#include <unistd.h>
2012-02-04 15:27:11 +02:00
#include <sys/time.h>
2018-09-02 01:50:22 +03:00
#include <fstream>
2021-12-23 00:38:49 +02:00
#include <functional>
#include <nlohmann/json.hpp>
#include <boost/container/small_vector.hpp>
2018-06-11 17:10:50 +03:00
#ifndef _WIN32 // TODO use portable implementation
# include <sys/resource.h>
#endif
#if HAVE_BOEHMGC
# define GC_INCLUDE_NEW
# include <gc/gc.h>
# include <gc/gc_cpp.h>
# include <gc/gc_allocator.h>
#endif
using json = nlohmann::json;
namespace nix {
static char * allocString(size_t size)
{
char * t;
#if HAVE_BOEHMGC
t = (char *) GC_MALLOC_ATOMIC(size);
#else
t = (char *) malloc(size);
#endif
if (!t) throw std::bad_alloc();
return t;
}
2015-03-19 15:11:35 +02:00
static char * dupString(const char * s)
{
char * t;
#if HAVE_BOEHMGC
t = GC_STRDUP(s);
2015-03-19 15:11:35 +02:00
#else
t = strdup(s);
#endif
if (!t) throw std::bad_alloc();
return t;
}
// When there's no need to write to the string, we can optimize away empty
// string allocations.
// This function handles makeImmutableString(std::string_view()) by returning
// the empty string.
static const char * makeImmutableString(std::string_view s)
{
const size_t size = s.size();
if (size == 0)
return "";
auto t = allocString(size + 1);
memcpy(t, s.data(), size);
t[size] = '\0';
return t;
}
RootValue allocRootValue(Value * v)
{
#if HAVE_BOEHMGC
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
#else
return std::make_shared<Value *>(v);
#endif
}
// Pretty print types for assertion errors
std::ostream & operator << (std::ostream & os, const ValueType t) {
os << showType(t);
return os;
}
std::string printValue(EvalState & state, Value & v)
{
std::ostringstream out;
v.print(state, out);
return out.str();
}
void Value::print(EvalState & state, std::ostream & str, PrintOptions options)
{
printValue(state, str, *this, options);
}
const Value * getPrimOp(const Value &v) {
const Value * primOp = &v;
while (primOp->isPrimOpApp()) {
primOp = primOp->payload.primOpApp.left;
}
assert(primOp->isPrimOp());
return primOp;
}
2023-06-21 23:06:16 +03:00
std::string_view showType(ValueType type, bool withArticle)
{
2023-06-21 23:06:16 +03:00
#define WA(a, w) withArticle ? a " " w : w
switch (type) {
2023-06-21 23:06:16 +03:00
case nInt: return WA("an", "integer");
case nBool: return WA("a", "Boolean");
case nString: return WA("a", "string");
case nPath: return WA("a", "path");
case nNull: return "null";
2023-06-21 23:06:16 +03:00
case nAttrs: return WA("a", "set");
case nList: return WA("a", "list");
case nFunction: return WA("a", "function");
case nExternal: return WA("an", "external value");
case nFloat: return WA("a", "float");
case nThunk: return WA("a", "thunk");
}
abort();
}
std::string showType(const Value & v)
{
2023-04-03 19:15:12 +03:00
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (v.internalType) {
case tString: return v.payload.string.context ? "a string with context" : "a string";
case tPrimOp:
return fmt("the built-in function '%s'", std::string(v.payload.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->payload.primOp->name));
case tExternal: return v.external()->showType();
case tThunk: return v.isBlackhole() ? "a black hole" : "a thunk";
case tApp: return "a function application";
default:
return std::string(showType(v.type()));
}
2023-04-03 19:15:12 +03:00
#pragma GCC diagnostic pop
}
PosIdx Value::determinePos(const PosIdx pos) const
{
2023-04-03 19:15:12 +03:00
// Allow selecting a subset of enum values
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (internalType) {
case tAttrs: return attrs()->pos;
case tLambda: return payload.lambda.fun->pos;
case tApp: return payload.app.left->determinePos(pos);
default: return pos;
}
2023-04-03 19:15:12 +03:00
#pragma GCC diagnostic pop
}
bool Value::isTrivial() const
{
return
internalType != tApp
&& internalType != tPrimOpApp
&& (internalType != tThunk
|| (dynamic_cast<ExprAttrs *>(payload.thunk.expr)
&& ((ExprAttrs *) payload.thunk.expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(payload.thunk.expr)
|| dynamic_cast<ExprList *>(payload.thunk.expr));
}
2014-04-04 22:14:11 +03:00
static Symbol getName(const AttrName & name, EvalState & state, Env & env)
{
if (name.symbol) {
return name.symbol;
} else {
Value nameValue;
name.expr->eval(state, env, nameValue);
state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name");
return state.symbols.create(nameValue.string_view());
}
}
static constexpr size_t BASE_ENV_SIZE = 128;
2021-06-29 22:09:48 +03:00
EvalState::EvalState(
const LookupPath & _lookupPath,
2021-06-29 22:09:48 +03:00
ref<Store> store,
const EvalSettings & settings,
2021-06-29 22:09:48 +03:00
std::shared_ptr<Store> buildStore)
: settings{settings}
, sWith(symbols.create("<with>"))
, sOutPath(symbols.create("outPath"))
, sDrvPath(symbols.create("drvPath"))
, sType(symbols.create("type"))
, sMeta(symbols.create("meta"))
, sName(symbols.create("name"))
2013-10-28 08:34:44 +02:00
, sValue(symbols.create("value"))
, sSystem(symbols.create("system"))
, sOverrides(symbols.create("__overrides"))
, sOutputs(symbols.create("outputs"))
, sOutputName(symbols.create("outputName"))
, sIgnoreNulls(symbols.create("__ignoreNulls"))
, sFile(symbols.create("file"))
, sLine(symbols.create("line"))
, sColumn(symbols.create("column"))
, sFunctor(symbols.create("__functor"))
, sToString(symbols.create("__toString"))
, sRight(symbols.create("right"))
, sWrong(symbols.create("wrong"))
Add support for passing structured data to builders Previously, all derivation attributes had to be coerced into strings so that they could be passed via the environment. This is lossy (e.g. lists get flattened, necessitating configureFlags vs. configureFlagsArray, of which the latter cannot be specified as an attribute), doesn't support attribute sets at all, and has size limitations (necessitating hacks like passAsFile). This patch adds a new mode for passing attributes to builders, namely encoded as a JSON file ".attrs.json" in the current directory of the builder. This mode is activated via the special attribute __structuredAttrs = true; (The idea is that one day we can set this in stdenv.mkDerivation.) For example, stdenv.mkDerivation { __structuredAttrs = true; name = "foo"; buildInputs = [ pkgs.hello pkgs.cowsay ]; doCheck = true; hardening.format = false; } results in a ".attrs.json" file containing (sans the indentation): { "buildInputs": [], "builder": "/nix/store/ygl61ycpr2vjqrx775l1r2mw1g2rb754-bash-4.3-p48/bin/bash", "configureFlags": [ "--with-foo", "--with-bar=1 2" ], "doCheck": true, "hardening": { "format": false }, "name": "foo", "nativeBuildInputs": [ "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10", "/nix/store/4jnvjin0r6wp6cv1hdm5jbkx3vinlcvk-cowsay-3.03" ], "propagatedBuildInputs": [], "propagatedNativeBuildInputs": [], "stdenv": "/nix/store/f3hw3p8armnzy6xhd4h8s7anfjrs15n2-stdenv", "system": "x86_64-linux" } "passAsFile" is ignored in this mode because it's not needed - large strings are included directly in the JSON representation. It is up to the builder to do something with the JSON representation. For example, in bash-based builders, lists/attrsets of string values could be mapped to bash (associative) arrays.
2017-01-25 17:42:07 +02:00
, sStructuredAttrs(symbols.create("__structuredAttrs"))
, sAllowedReferences(symbols.create("allowedReferences"))
, sAllowedRequisites(symbols.create("allowedRequisites"))
, sDisallowedReferences(symbols.create("disallowedReferences"))
, sDisallowedRequisites(symbols.create("disallowedRequisites"))
, sMaxSize(symbols.create("maxSize"))
, sMaxClosureSize(symbols.create("maxClosureSize"))
Add support for passing structured data to builders Previously, all derivation attributes had to be coerced into strings so that they could be passed via the environment. This is lossy (e.g. lists get flattened, necessitating configureFlags vs. configureFlagsArray, of which the latter cannot be specified as an attribute), doesn't support attribute sets at all, and has size limitations (necessitating hacks like passAsFile). This patch adds a new mode for passing attributes to builders, namely encoded as a JSON file ".attrs.json" in the current directory of the builder. This mode is activated via the special attribute __structuredAttrs = true; (The idea is that one day we can set this in stdenv.mkDerivation.) For example, stdenv.mkDerivation { __structuredAttrs = true; name = "foo"; buildInputs = [ pkgs.hello pkgs.cowsay ]; doCheck = true; hardening.format = false; } results in a ".attrs.json" file containing (sans the indentation): { "buildInputs": [], "builder": "/nix/store/ygl61ycpr2vjqrx775l1r2mw1g2rb754-bash-4.3-p48/bin/bash", "configureFlags": [ "--with-foo", "--with-bar=1 2" ], "doCheck": true, "hardening": { "format": false }, "name": "foo", "nativeBuildInputs": [ "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10", "/nix/store/4jnvjin0r6wp6cv1hdm5jbkx3vinlcvk-cowsay-3.03" ], "propagatedBuildInputs": [], "propagatedNativeBuildInputs": [], "stdenv": "/nix/store/f3hw3p8armnzy6xhd4h8s7anfjrs15n2-stdenv", "system": "x86_64-linux" } "passAsFile" is ignored in this mode because it's not needed - large strings are included directly in the JSON representation. It is up to the builder to do something with the JSON representation. For example, in bash-based builders, lists/attrsets of string values could be mapped to bash (associative) arrays.
2017-01-25 17:42:07 +02:00
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
, sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
2020-06-18 14:44:40 +03:00
, sRecurseForDerivations(symbols.create("recurseForDerivations"))
2018-11-29 20:18:36 +02:00
, sDescription(symbols.create("description"))
, sSelf(symbols.create("self"))
, sEpsilon(symbols.create(""))
, sStartSet(symbols.create("startSet"))
, sOperator(symbols.create("operator"))
, sKey(symbols.create("key"))
, sPath(symbols.create("path"))
, sPrefix(symbols.create("prefix"))
, sOutputSpecified(symbols.create("outputSpecified"))
, exprSymbols{
.sub = symbols.create("__sub"),
.lessThan = symbols.create("__lessThan"),
.mul = symbols.create("__mul"),
.div = symbols.create("__div"),
.or_ = symbols.create("or"),
.findFile = symbols.create("__findFile"),
.nixPath = symbols.create("__nixPath"),
.body = symbols.create("body"),
}
, repair(NoRepair)
2022-01-04 20:23:11 +02:00
, emptyBindings(0)
, rootFS(
settings.restrictEval || settings.pureEval
? ref<SourceAccessor>(AllowListSourceAccessor::create(getFSSourceAccessor(), {},
[&settings](const CanonPath & path) -> RestrictedPathError {
auto modeInformation = settings.pureEval
? "in pure evaluation mode (use '--impure' to override)"
: "in restricted mode";
throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation);
}))
: getFSSourceAccessor())
, corepkgsFS(make_ref<MemorySourceAccessor>())
, internalFS(make_ref<MemorySourceAccessor>())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store)
2021-06-29 22:09:48 +03:00
, buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr)
, debugStop(false)
, trylevel(0)
, regexCache(makeRegexCache())
#if HAVE_BOEHMGC
, valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
, env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
, baseEnvP(std::allocate_shared<Env *>(traceable_allocator<Env *>(), &allocEnv(BASE_ENV_SIZE)))
, baseEnv(**baseEnvP)
#else
, baseEnv(allocEnv(BASE_ENV_SIZE))
#endif
, staticBaseEnv{std::make_shared<StaticEnv>(nullptr, nullptr)}
{
corepkgsFS->setPathDisplay("<nix", ">");
internalFS->setPathDisplay("«nix-internal»", "");
countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
assertGCInitialized();
2021-10-22 23:27:04 +03:00
static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes");
vEmptyList.mkList(buildList(0));
vNull.mkNull();
2024-03-21 00:06:23 +02:00
vTrue.mkBool(true);
vFalse.mkBool(false);
vStringRegular.mkString("regular");
vStringDirectory.mkString("directory");
vStringSymlink.mkString("symlink");
vStringUnknown.mkString("unknown");
/* Initialise the Nix expression search path. */
if (!settings.pureEval) {
for (auto & i : _lookupPath.elements)
lookupPath.elements.emplace_back(LookupPath::Elem {i});
for (auto & i : settings.nixPath.get())
lookupPath.elements.emplace_back(LookupPath::Elem::parse(i));
}
/* Allow access to all paths in the search path. */
if (rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
for (auto & i : lookupPath.elements)
resolveLookupPathPath(i.path, true);
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv();
}
EvalState::~EvalState()
{
}
2021-10-07 13:11:00 +03:00
void EvalState::allowPath(const Path & path)
{
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->allowPrefix(CanonPath(path));
2021-10-07 13:11:00 +03:00
}
void EvalState::allowPath(const StorePath & storePath)
{
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath)));
}
void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v)
{
allowPath(storePath);
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
mkStorePathString(storePath, v);
}
inline static bool isJustSchemePrefix(std::string_view prefix)
{
return
!prefix.empty()
&& prefix[prefix.size() - 1] == ':'
&& isValidSchemeName(prefix.substr(0, prefix.size() - 1));
}
bool isAllowedURI(std::string_view uri, const Strings & allowedUris)
{
/* 'uri' should be equal to a prefix, or in a subdirectory of a
prefix. Thus, the prefix https://github.co does not permit
2023-12-06 13:43:20 +02:00
access to https://github.com. */
for (auto & prefix : allowedUris) {
2023-12-06 15:08:22 +02:00
if (uri == prefix
// Allow access to subdirectories of the prefix.
|| (uri.size() > prefix.size()
&& prefix.size() > 0
&& hasPrefix(uri, prefix)
&& (
// Allow access to subdirectories of the prefix.
2023-12-06 15:08:22 +02:00
prefix[prefix.size() - 1] == '/'
|| uri[prefix.size()] == '/'
// Allow access to whole schemes
|| isJustSchemePrefix(prefix)
)
))
return true;
}
return false;
}
void EvalState::checkURI(const std::string & uri)
{
if (!settings.restrictEval) return;
if (isAllowedURI(uri, settings.allowedUris.get())) return;
/* If the URI is a path, then check it against allowedPaths as
well. */
if (hasPrefix(uri, "/")) {
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->checkAccess(CanonPath(uri));
return;
}
if (hasPrefix(uri, "file://")) {
if (auto rootFS2 = rootFS.dynamic_pointer_cast<AllowListSourceAccessor>())
rootFS2->checkAccess(CanonPath(uri.substr(7)));
return;
}
throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri);
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
Path EvalState::toRealPath(const Path & path, const NixStringContext & context)
{
// FIXME: check whether 'path' is in 'context'.
return
!context.empty() && store->isInStore(path)
? store->toRealPath(path)
: path;
2019-11-10 18:23:35 +02:00
}
Value * EvalState::addConstant(const std::string & name, Value & v, Constant info)
2010-03-30 17:39:27 +03:00
{
Value * v2 = allocValue();
*v2 = v;
addConstant(name, v2, info);
return v2;
2010-03-30 17:39:27 +03:00
}
void EvalState::addConstant(const std::string & name, Value * v, Constant info)
2020-03-02 19:15:06 +02:00
{
auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name;
2010-03-30 17:39:27 +03:00
constantInfos.push_back({name2, info});
2010-03-30 17:39:27 +03:00
if (!(settings.pureEval && info.impureOnly)) {
/* Check the type, if possible.
We might know the type of a thunk in advance, so be allowed
to just write it down in that case. */
if (auto gotType = v->type(true); gotType != nThunk)
assert(info.type == gotType);
/* Install value the base environment. */
staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(name2), v));
}
}
2023-11-16 12:10:25 +02:00
void PrimOp::check()
{
if (arity > maxPrimOpArity) {
throw Error("primop arity must not exceed %1%", maxPrimOpArity);
}
}
std::ostream & operator<<(std::ostream & output, const PrimOp & primOp)
{
output << "primop " << primOp.name;
return output;
}
const PrimOp * Value::primOpAppPrimOp() const
{
Value * left = payload.primOpApp.left;
while (left && !left->isPrimOp()) {
left = left->payload.primOpApp.left;
}
if (!left)
return nullptr;
return left->primOp();
}
2023-11-16 12:10:25 +02:00
void Value::mkPrimOp(PrimOp * p)
{
p->check();
finishValue(tPrimOp, { .primOp = p });
2023-11-16 12:10:25 +02:00
}
Value * EvalState::addPrimOp(PrimOp && primOp)
{
/* Hack to make constants lazy: turn them into a application of
the primop to a dummy value. */
if (primOp.arity == 0) {
primOp.arity = 1;
auto vPrimOp = allocValue();
vPrimOp->mkPrimOp(new PrimOp(primOp));
Value v;
2022-01-04 19:40:39 +02:00
v.mkApp(vPrimOp, vPrimOp);
return addConstant(primOp.name, v, {
.type = nThunk, // FIXME
.doc = primOp.doc,
});
}
auto envName = symbols.create(primOp.name);
if (hasPrefix(primOp.name, "__"))
primOp.name = primOp.name.substr(2);
Value * v = allocValue();
v->mkPrimOp(new PrimOp(primOp));
2021-11-25 17:53:59 +02:00
staticBaseEnv->vars.emplace_back(envName, baseEnvDispl);
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->payload.attrs->push_back(Attr(symbols.create(primOp.name), v));
return v;
}
Value & EvalState::getBuiltin(const std::string & name)
{
return *baseEnv.values[0]->attrs()->find(symbols.create(name))->value;
}
2020-08-25 14:31:11 +03:00
std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
2020-08-25 14:31:11 +03:00
auto v2 = &v;
if (auto * doc = v2->primOp()->doc)
2020-08-25 14:31:11 +03:00
return Doc {
.pos = {},
.name = v2->primOp()->name,
.arity = v2->primOp()->arity,
.args = v2->primOp()->args,
.doc = doc,
2020-08-25 14:31:11 +03:00
};
}
return {};
}
2022-03-31 18:37:36 +03:00
// just for the current level of StaticEnv, not the whole chain.
void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se)
2021-10-02 22:47:36 +03:00
{
2022-03-31 18:37:36 +03:00
std::cout << ANSI_MAGENTA;
for (auto & i : se.vars)
std::cout << st[i.first] << " ";
2022-03-31 18:37:36 +03:00
std::cout << ANSI_NORMAL;
std::cout << std::endl;
}
2022-01-05 21:28:31 +02:00
2022-03-31 18:37:36 +03:00
// just for the current level of Env, not the whole chain.
void printWithBindings(const SymbolTable & st, const Env & env)
2022-03-31 18:37:36 +03:00
{
if (!env.values[0]->isThunk()) {
2022-04-29 20:24:54 +03:00
std::cout << "with: ";
2022-01-05 21:28:31 +02:00
std::cout << ANSI_MAGENTA;
auto j = env.values[0]->attrs()->begin();
while (j != env.values[0]->attrs()->end()) {
std::cout << st[j->name] << " ";
2022-03-31 18:37:36 +03:00
++j;
2022-01-04 03:29:43 +02:00
}
2022-01-05 21:28:31 +02:00
std::cout << ANSI_NORMAL;
2022-01-04 03:29:43 +02:00
std::cout << std::endl;
2022-03-31 18:37:36 +03:00
}
}
void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, int lvl)
2022-03-31 18:37:36 +03:00
{
std::cout << "Env level " << lvl << std::endl;
2022-01-04 03:29:43 +02:00
2022-03-31 18:37:36 +03:00
if (se.up && env.up) {
2022-04-29 20:24:54 +03:00
std::cout << "static: ";
printStaticEnvBindings(st, se);
if (se.isWith)
printWithBindings(st, env);
2022-03-31 18:37:36 +03:00
std::cout << std::endl;
printEnvBindings(st, *se.up, *env.up, ++lvl);
} else {
2022-01-05 21:28:31 +02:00
std::cout << ANSI_MAGENTA;
// for the top level, don't print the double underscore ones;
// they are in builtins.
for (auto & i : se.vars)
if (!hasPrefix(st[i.first], "__"))
std::cout << st[i.first] << " ";
2022-01-05 21:28:31 +02:00
std::cout << ANSI_NORMAL;
2022-01-04 03:29:43 +02:00
std::cout << std::endl;
if (se.isWith)
printWithBindings(st, env); // probably nothing there for the top level.
2022-01-04 03:29:43 +02:00
std::cout << std::endl;
}
2021-10-02 22:47:36 +03:00
}
void printEnvBindings(const EvalState &es, const Expr & expr, const Env & env)
2021-10-02 22:47:36 +03:00
{
2021-11-25 17:23:07 +02:00
// just print the names for now
auto se = es.getStaticEnv(expr);
if (se)
printEnvBindings(es.symbols, *se, env, 0);
}
2022-05-06 00:43:23 +03:00
void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, ValMap & vm)
{
// add bindings for the next level up first, so that the bindings for this level
// override the higher levels.
// The top level bindings (builtins) are skipped since they are added for us by initEnv()
if (env.up && se.up) {
mapStaticEnvBindings(st, *se.up, *env.up, vm);
if (se.isWith && !env.values[0]->isThunk()) {
// add 'with' bindings.
for (auto & j : *env.values[0]->attrs())
vm[st[j.name]] = j.value;
} else {
// iterate through staticenv bindings and add them.
for (auto & i : se.vars)
vm[st[i.first]] = env.values[i.second];
}
2021-11-25 17:23:07 +02:00
}
2021-10-02 22:47:36 +03:00
}
2022-05-06 00:43:23 +03:00
std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env)
{
2022-05-06 00:43:23 +03:00
auto vm = std::make_unique<ValMap>();
mapStaticEnvBindings(st, se, env, *vm);
2021-10-12 01:32:43 +03:00
return vm;
}
2024-02-03 06:07:42 +02:00
/**
* Sets `inDebugger` to true on construction and false on destruction.
*/
class DebuggerGuard {
bool & inDebugger;
public:
DebuggerGuard(bool & inDebugger) : inDebugger(inDebugger) {
inDebugger = true;
}
~DebuggerGuard() {
inDebugger = false;
}
};
bool EvalState::canDebug()
{
return debugRepl && !debugTraces.empty();
}
void EvalState::runDebugRepl(const Error * error)
{
if (!canDebug())
return;
assert(!debugTraces.empty());
const DebugTrace & last = debugTraces.front();
const Env & env = last.env;
const Expr & expr = last.expr;
runDebugRepl(error, env, expr);
}
2022-05-20 19:33:50 +03:00
void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr)
2022-05-20 02:01:23 +03:00
{
2024-02-03 06:07:42 +02:00
// Make sure we have a debugger to run and we're not already in a debugger.
if (!debugRepl || inDebugger)
2022-05-20 19:33:50 +03:00
return;
2022-05-25 13:32:22 +03:00
2022-05-20 02:01:23 +03:00
auto dts =
error && expr.getPos()
? std::make_unique<DebugTraceStacker>(
*this,
DebugTrace {
.pos = error->info().pos ? error->info().pos : positions[expr.getPos()],
2022-05-20 02:01:23 +03:00
.expr = expr,
.env = env,
.hint = error->info().msg,
.isError = true
})
: nullptr;
if (error)
{
2024-02-03 05:58:35 +02:00
printError("%s\n", error->what());
if (trylevel > 0 && error->info().level != lvlInfo)
2022-07-11 19:21:12 +03:00
printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n");
}
2022-05-20 02:01:23 +03:00
auto se = getStaticEnv(expr);
if (se) {
auto vm = mapStaticEnvBindings(symbols, *se.get(), env);
2024-02-03 06:07:42 +02:00
DebuggerGuard _guard(inDebugger);
auto exitStatus = (debugRepl)(ref<EvalState>(shared_from_this()), *vm);
switch (exitStatus) {
case ReplExitStatus::QuitAll:
if (error)
throw *error;
throw Exit(0);
case ReplExitStatus::Continue:
break;
default:
abort();
}
2022-05-20 02:01:23 +03:00
}
}
2024-02-23 03:18:27 +02:00
template<typename... Args>
void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const
{
2024-02-23 03:18:27 +02:00
e.addTrace(nullptr, HintFmt(formatArgs...));
}
2024-02-23 03:18:27 +02:00
template<typename... Args>
void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const
{
2024-02-23 03:18:27 +02:00
e.addTrace(positions[pos], HintFmt(formatArgs...));
}
template<typename... Args>
static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
EvalState & state,
Expr & expr,
Env & env,
std::shared_ptr<Pos> && pos,
const Args & ... formatArgs)
2021-12-23 18:08:41 +02:00
{
return std::make_unique<DebugTraceStacker>(state,
DebugTrace {
.pos = std::move(pos),
.expr = expr,
.env = env,
2024-02-04 06:35:19 +02:00
.hint = HintFmt(formatArgs...),
.isError = false
});
2021-12-23 18:08:41 +02:00
}
2021-12-23 00:38:49 +02:00
DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
: evalState(evalState)
, trace(std::move(t))
2022-01-08 20:03:48 +02:00
{
evalState.debugTraces.push_front(trace);
2022-05-23 06:45:24 +03:00
if (evalState.debugStop && evalState.debugRepl)
2022-05-20 19:33:50 +03:00
evalState.runDebugRepl(nullptr, trace.env, trace.expr);
2022-01-08 20:03:48 +02:00
}
2021-12-23 00:38:49 +02:00
void Value::mkString(std::string_view s)
2010-03-30 21:05:54 +03:00
{
mkString(makeImmutableString(s));
2010-03-30 21:05:54 +03:00
}
static const char * * encodeContext(const NixStringContext & context)
2010-03-30 21:05:54 +03:00
{
if (!context.empty()) {
2018-05-02 14:56:34 +03:00
size_t n = 0;
auto ctx = (const char * *)
2015-03-19 15:11:35 +02:00
allocBytes((context.size() + 1) * sizeof(char *));
2015-07-17 20:24:28 +03:00
for (auto & i : context)
ctx[n++] = dupString(i.to_string().c_str());
ctx[n] = 0;
return ctx;
} else
return nullptr;
2010-03-30 21:05:54 +03:00
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
void Value::mkString(std::string_view s, const NixStringContext & context)
{
mkString(makeImmutableString(s), encodeContext(context));
}
2010-03-30 21:05:54 +03:00
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
void Value::mkStringMove(const char * s, const NixStringContext & context)
2010-03-30 21:05:54 +03:00
{
mkString(s, encodeContext(context));
}
void Value::mkPath(const SourcePath & path)
2010-03-30 21:05:54 +03:00
{
mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
2010-03-30 21:05:54 +03:00
}
2013-10-08 15:24:53 +03:00
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
{
2020-02-24 15:33:01 +02:00
for (auto l = var.level; l; --l, env = env->up) ;
if (!var.fromWith) return env->values[var.displ];
// This early exit defeats the `maybeThunk` optimization for variables from `with`,
// The added complexity of handling this appears to be similarly in cost, or
// the cases where applicable were insignificant in the first place.
if (noEval) return nullptr;
auto * fromWith = var.fromWith;
while (1) {
forceAttrs(*env->values[0], fromWith->pos, "while evaluating the first subexpression of a with expression");
if (auto j = env->values[0]->attrs()->get(var.name)) {
if (countCalls) attrSelects[j->pos]++;
return j->value;
}
if (!fromWith->parentWith)
error<UndefinedVarError>("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow();
for (size_t l = fromWith->prevWith; l; --l, env = env->up) ;
fromWith = fromWith->parentWith;
}
}
ListBuilder::ListBuilder(EvalState & state, size_t size)
: size(size)
, elems(size <= 2 ? inlineElems : (Value * *) allocBytes(size * sizeof(Value *)))
2010-03-30 17:39:27 +03:00
{
state.nrListElems += size;
2010-03-30 17:39:27 +03:00
}
2024-03-21 00:11:54 +02:00
Value * EvalState::getBool(bool b) {
return b ? &vTrue : &vFalse;
}
2010-03-30 17:39:27 +03:00
unsigned long nrThunks = 0;
static inline void mkThunk(Value & v, Env & env, Expr * expr)
{
v.mkThunk(&env, expr);
nrThunks++;
}
void EvalState::mkThunk_(Value & v, Expr * expr)
{
mkThunk(v, baseEnv, expr);
}
void EvalState::mkPos(Value & v, PosIdx p)
{
use byte indexed locations for PosIdx we now keep not a table of all positions, but a table of all origins and their sizes. position indices are now direct pointers into the virtual concatenation of all parsed contents. this slightly reduces memory usage and time spent in the parser, at the cost of not being able to report positions if the total input size exceeds 4GiB. this limit is not unique to nix though, rustc and clang also limit their input to 4GiB (although at least clang refuses to process inputs that are larger, we will not). this new 4GiB limit probably will not cause any problems for quite a while, all of nixpkgs together is less than 100MiB in size and already needs over 700MiB of memory and multiple seconds just to parse. 4GiB worth of input will easily take multiple minutes and over 30GiB of memory without even evaluating anything. if problems *do* arise we can probably recover the old table-based system by adding some tracking to Pos::Origin (or increasing the size of PosIdx outright), but for time being this looks like more complexity than it's worth. since we now need to read the entire input again to determine the line/column of a position we'll make unsafeGetAttrPos slightly lazy: mostly the set it returns is only used to determine the file of origin of an attribute, not its exact location. the thunks do not add measurable runtime overhead. notably this change is necessary to allow changing the parser since apparently nothing supports nix's very idiosyncratic line ending choice of "anything goes", making it very hard to calculate line/column positions in the parser (while byte offsets are very easy).
2024-01-29 07:19:23 +02:00
auto origin = positions.originOf(p);
if (auto path = std::get_if<SourcePath>(&origin)) {
auto attrs = buildBindings(3);
2023-04-06 16:25:06 +03:00
attrs.alloc(sFile).mkString(path->path.abs());
use byte indexed locations for PosIdx we now keep not a table of all positions, but a table of all origins and their sizes. position indices are now direct pointers into the virtual concatenation of all parsed contents. this slightly reduces memory usage and time spent in the parser, at the cost of not being able to report positions if the total input size exceeds 4GiB. this limit is not unique to nix though, rustc and clang also limit their input to 4GiB (although at least clang refuses to process inputs that are larger, we will not). this new 4GiB limit probably will not cause any problems for quite a while, all of nixpkgs together is less than 100MiB in size and already needs over 700MiB of memory and multiple seconds just to parse. 4GiB worth of input will easily take multiple minutes and over 30GiB of memory without even evaluating anything. if problems *do* arise we can probably recover the old table-based system by adding some tracking to Pos::Origin (or increasing the size of PosIdx outright), but for time being this looks like more complexity than it's worth. since we now need to read the entire input again to determine the line/column of a position we'll make unsafeGetAttrPos slightly lazy: mostly the set it returns is only used to determine the file of origin of an attribute, not its exact location. the thunks do not add measurable runtime overhead. notably this change is necessary to allow changing the parser since apparently nothing supports nix's very idiosyncratic line ending choice of "anything goes", making it very hard to calculate line/column positions in the parser (while byte offsets are very easy).
2024-01-29 07:19:23 +02:00
makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn));
v.mkAttrs(attrs);
} else
2022-01-04 19:40:39 +02:00
v.mkNull();
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
void EvalState::mkStorePathString(const StorePath & p, Value & v)
{
v.mkString(
store->printStorePath(p),
NixStringContext {
NixStringContextElem::Opaque { .path = p },
});
}
std::string EvalState::mkOutputStringRaw(
const SingleDerivedPath::Built & b,
std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings)
{
/* In practice, this is testing for the case of CA derivations, or
dynamic derivations. */
return optStaticOutputPath
? store->printStorePath(std::move(*optStaticOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
}
void EvalState::mkOutputString(
Value & value,
const SingleDerivedPath::Built & b,
std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings)
{
value.mkString(
mkOutputStringRaw(b, optStaticOutputPath, xpSettings),
NixStringContext { b });
}
std::string EvalState::mkSingleDerivedPathStringRaw(
const SingleDerivedPath & p)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
return store->printStorePath(o.path);
},
[&](const SingleDerivedPath::Built & b) {
auto optStaticOutputPath = std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
auto drv = store->readDerivation(o.path);
auto i = drv.outputs.find(b.output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
return i->second.path(*store, drv.name, b.output);
},
[&](const SingleDerivedPath::Built & o) -> std::optional<StorePath> {
return std::nullopt;
},
}, b.drvPath->raw());
return mkOutputStringRaw(b, optStaticOutputPath);
}
}, p.raw());
}
void EvalState::mkSingleDerivedPathString(
const SingleDerivedPath & p,
Value & v)
{
v.mkString(
mkSingleDerivedPathStringRaw(p),
NixStringContext {
std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
});
}
/* Create a thunk for the delayed computation of the given expression
in the given environment. But if the expression is a variable,
then look it up right away. This significantly reduces the number
of thunks allocated. */
Value * Expr::maybeThunk(EvalState & state, Env & env)
{
Value * v = state.allocValue();
mkThunk(*v, env, this);
return v;
}
Value * ExprVar::maybeThunk(EvalState & state, Env & env)
{
2013-10-08 15:24:53 +03:00
Value * v = state.lookupVar(&env, *this, true);
/* The value might not be initialised in the environment yet.
In that case, ignore it. */
if (v) { state.nrAvoided++; return v; }
return Expr::maybeThunk(state, env);
}
Value * ExprString::maybeThunk(EvalState & state, Env & env)
{
state.nrAvoided++;
return &v;
}
Value * ExprInt::maybeThunk(EvalState & state, Env & env)
{
state.nrAvoided++;
return &v;
}
Value * ExprFloat::maybeThunk(EvalState & state, Env & env)
{
state.nrAvoided++;
return &v;
}
Value * ExprPath::maybeThunk(EvalState & state, Env & env)
{
state.nrAvoided++;
return &v;
}
void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
2010-03-30 12:22:33 +03:00
{
FileEvalCache::iterator i;
if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) {
v = i->second;
return;
}
auto resolvedPath = resolveExprPath(path);
if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) {
v = i->second;
return;
}
printTalkative("evaluating file '%1%'", resolvedPath);
Expr * e = nullptr;
auto j = fileParseCache.find(resolvedPath);
if (j != fileParseCache.end())
e = j->second;
if (!e)
e = parseExprFromFile(resolvedPath);
fileParseCache[resolvedPath] = e;
try {
2022-05-23 06:45:24 +03:00
auto dts = debugRepl
? makeDebugTraceStacker(
*this,
*e,
this->baseEnv,
e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
"while evaluating the file '%1%':", resolvedPath.to_string())
: nullptr;
2021-12-23 00:38:49 +02:00
// Enforce that 'flake.nix' is a direct attrset, not a
// computation.
if (mustBeTrivial &&
!(dynamic_cast<ExprAttrs *>(e)))
error<EvalError>("file '%s' must be an attribute set", path).debugThrow();
eval(e, v);
} catch (Error & e) {
addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string());
throw;
}
fileEvalCache[resolvedPath] = v;
if (path != resolvedPath) fileEvalCache[path] = v;
2013-09-02 19:34:04 +03:00
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v)
{
e->eval(*this, baseEnv, v);
}
inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx)
2010-04-16 18:13:47 +03:00
{
try {
Value v;
e->eval(*this, env, v);
if (v.type() != nBool)
error<TypeError>(
"expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).withFrame(env, *e).debugThrow();
return v.boolean();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
}
inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx)
{
try {
e->eval(*this, env, v);
if (v.type() != nAttrs)
error<TypeError>(
"expected a set but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).withFrame(env, *e).debugThrow();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
2010-04-16 18:13:47 +03:00
}
2010-04-13 01:03:27 +03:00
void Expr::eval(EvalState & state, Env & env, Value & v)
{
abort();
}
void ExprInt::eval(EvalState & state, Env & env, Value & v)
{
v = this->v;
}
void ExprFloat::eval(EvalState & state, Env & env, Value & v)
{
v = this->v;
}
void ExprString::eval(EvalState & state, Env & env, Value & v)
{
v = this->v;
}
void ExprPath::eval(EvalState & state, Env & env, Value & v)
{
v = this->v;
}
Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
{
Env & inheritEnv = state.allocEnv(inheritFromExprs->size());
inheritEnv.up = &up;
Displacement displ = 0;
for (auto from : *inheritFromExprs)
inheritEnv.values[displ++] = from->maybeThunk(state, up);
return &inheritEnv;
}
void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{
auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
2022-01-04 21:29:17 +02:00
auto dynamicEnv = &env;
bool sort = false;
2010-04-15 02:25:05 +03:00
if (recursive) {
/* Create a new environment that contains the attributes in
this `rec'. */
Env & env2(state.allocEnv(attrs.size()));
env2.up = &env;
Dynamic attrs This adds new syntax for attribute names: * attrs."${name}" => getAttr name attrs * attrs ? "${name}" => isAttrs attrs && hasAttr attrs name * attrs."${name}" or def => if attrs ? "${name}" then attrs."${name}" else def * { "${name}" = value; } => listToAttrs [{ inherit name value; }] Of course, it's a bit more complicated than that. The attribute chains can be arbitrarily long and contain combinations of static and dynamic parts (e.g. attrs."${foo}".bar."${baz}" or qux), which is relatively straightforward for the getAttrs/hasAttrs cases but is more complex for the listToAttrs case due to rules about duplicate attribute definitions. For attribute sets with dynamic attribute names, duplicate static attributes are detected at parse time while duplicate dynamic attributes are detected when the attribute set is forced. So, for example, { a = null; a.b = null; "${"c"}" = true; } will be a parse-time error, while { a = {}; "${"a"}".b = null; c = true; } will be an eval-time error (technically that case could theoretically be detected at parse time, but the general case would require full evaluation). Moreover, duplicate dynamic attributes are not allowed even in cases where they would be with static attributes ({ a.b.d = true; a.b.c = false; } is legal, but { a."${"b"}".d = true; a."${"b"}".c = false; } is not). This restriction might be relaxed in the future in cases where the static variant would not be an error, but it is not obvious that that is desirable. Finally, recursive attribute sets with dynamic attributes have the static attributes in scope but not the dynamic ones. So rec { a = true; "${"b"}" = a; } is equivalent to { a = true; b = true; } but rec { "${"a"}" = true; b = a; } would be an error or use a from the surrounding scope if it exists. Note that the getAttr, getAttr or default, and hasAttr are all implemented purely in the parser as syntactic sugar, while attribute sets with dynamic attribute names required changes to the AST to be implemented cleanly. This is an alternative solution to and closes #167 Signed-off-by: Shea Levy <shea@shealevy.com>
2013-09-21 06:25:30 +03:00
dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
AttrDefs::iterator overrides = attrs.find(state.sOverrides);
bool hasOverrides = overrides != attrs.end();
/* The recursive attributes are evaluated in the new
environment, while the inherited attributes are evaluated
in the original environment. */
2020-02-24 15:33:01 +02:00
Displacement displ = 0;
2015-07-17 20:24:28 +03:00
for (auto & i : attrs) {
Value * vAttr;
if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
vAttr = state.allocValue();
mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e);
} else
vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
env2.values[displ++] = vAttr;
bindings.insert(i.first, vAttr, i.second.pos);
}
/* If the rec contains an attribute called `__overrides', then
evaluate it, and add the attributes in that set to the rec.
This allows overriding of recursive attributes, which is
otherwise not possible. (You can use the // operator to
replace an attribute, but other attributes in the rec will
still reference the original value, because that value has
been substituted into the bodies of the other attributes.
Hence we need __overrides.) */
if (hasOverrides) {
Value * vOverrides = (*bindings.bindings)[overrides->second.displ].value;
state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute");
bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size()));
for (auto & i : *vOverrides->attrs()) {
AttrDefs::iterator j = attrs.find(i.name);
if (j != attrs.end()) {
(*bindings.bindings)[j->second.displ] = i;
env2.values[j->second.displ] = i.value;
} else
bindings.push_back(i);
}
sort = true;
}
}
else {
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
for (auto & i : attrs)
bindings.insert(
i.first,
i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)),
i.second.pos);
}
Dynamic attrs This adds new syntax for attribute names: * attrs."${name}" => getAttr name attrs * attrs ? "${name}" => isAttrs attrs && hasAttr attrs name * attrs."${name}" or def => if attrs ? "${name}" then attrs."${name}" else def * { "${name}" = value; } => listToAttrs [{ inherit name value; }] Of course, it's a bit more complicated than that. The attribute chains can be arbitrarily long and contain combinations of static and dynamic parts (e.g. attrs."${foo}".bar."${baz}" or qux), which is relatively straightforward for the getAttrs/hasAttrs cases but is more complex for the listToAttrs case due to rules about duplicate attribute definitions. For attribute sets with dynamic attribute names, duplicate static attributes are detected at parse time while duplicate dynamic attributes are detected when the attribute set is forced. So, for example, { a = null; a.b = null; "${"c"}" = true; } will be a parse-time error, while { a = {}; "${"a"}".b = null; c = true; } will be an eval-time error (technically that case could theoretically be detected at parse time, but the general case would require full evaluation). Moreover, duplicate dynamic attributes are not allowed even in cases where they would be with static attributes ({ a.b.d = true; a.b.c = false; } is legal, but { a."${"b"}".d = true; a."${"b"}".c = false; } is not). This restriction might be relaxed in the future in cases where the static variant would not be an error, but it is not obvious that that is desirable. Finally, recursive attribute sets with dynamic attributes have the static attributes in scope but not the dynamic ones. So rec { a = true; "${"b"}" = a; } is equivalent to { a = true; b = true; } but rec { "${"a"}" = true; b = a; } would be an error or use a from the surrounding scope if it exists. Note that the getAttr, getAttr or default, and hasAttr are all implemented purely in the parser as syntactic sugar, while attribute sets with dynamic attribute names required changes to the AST to be implemented cleanly. This is an alternative solution to and closes #167 Signed-off-by: Shea Levy <shea@shealevy.com>
2013-09-21 06:25:30 +03:00
/* Dynamic attrs apply *after* rec and __overrides. */
2015-07-17 20:24:28 +03:00
for (auto & i : dynamicAttrs) {
Dynamic attrs This adds new syntax for attribute names: * attrs."${name}" => getAttr name attrs * attrs ? "${name}" => isAttrs attrs && hasAttr attrs name * attrs."${name}" or def => if attrs ? "${name}" then attrs."${name}" else def * { "${name}" = value; } => listToAttrs [{ inherit name value; }] Of course, it's a bit more complicated than that. The attribute chains can be arbitrarily long and contain combinations of static and dynamic parts (e.g. attrs."${foo}".bar."${baz}" or qux), which is relatively straightforward for the getAttrs/hasAttrs cases but is more complex for the listToAttrs case due to rules about duplicate attribute definitions. For attribute sets with dynamic attribute names, duplicate static attributes are detected at parse time while duplicate dynamic attributes are detected when the attribute set is forced. So, for example, { a = null; a.b = null; "${"c"}" = true; } will be a parse-time error, while { a = {}; "${"a"}".b = null; c = true; } will be an eval-time error (technically that case could theoretically be detected at parse time, but the general case would require full evaluation). Moreover, duplicate dynamic attributes are not allowed even in cases where they would be with static attributes ({ a.b.d = true; a.b.c = false; } is legal, but { a."${"b"}".d = true; a."${"b"}".c = false; } is not). This restriction might be relaxed in the future in cases where the static variant would not be an error, but it is not obvious that that is desirable. Finally, recursive attribute sets with dynamic attributes have the static attributes in scope but not the dynamic ones. So rec { a = true; "${"b"}" = a; } is equivalent to { a = true; b = true; } but rec { "${"a"}" = true; b = a; } would be an error or use a from the surrounding scope if it exists. Note that the getAttr, getAttr or default, and hasAttr are all implemented purely in the parser as syntactic sugar, while attribute sets with dynamic attribute names required changes to the AST to be implemented cleanly. This is an alternative solution to and closes #167 Signed-off-by: Shea Levy <shea@shealevy.com>
2013-09-21 06:25:30 +03:00
Value nameVal;
2015-07-17 20:24:28 +03:00
i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos);
if (nameVal.type() == nNull)
continue;
state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute");
auto nameSym = state.symbols.create(nameVal.string_view());
if (sort)
// FIXME: inefficient
bindings.bindings->sort();
if (auto j = bindings.bindings->get(nameSym))
state.error<EvalError>("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow();
Dynamic attrs This adds new syntax for attribute names: * attrs."${name}" => getAttr name attrs * attrs ? "${name}" => isAttrs attrs && hasAttr attrs name * attrs."${name}" or def => if attrs ? "${name}" then attrs."${name}" else def * { "${name}" = value; } => listToAttrs [{ inherit name value; }] Of course, it's a bit more complicated than that. The attribute chains can be arbitrarily long and contain combinations of static and dynamic parts (e.g. attrs."${foo}".bar."${baz}" or qux), which is relatively straightforward for the getAttrs/hasAttrs cases but is more complex for the listToAttrs case due to rules about duplicate attribute definitions. For attribute sets with dynamic attribute names, duplicate static attributes are detected at parse time while duplicate dynamic attributes are detected when the attribute set is forced. So, for example, { a = null; a.b = null; "${"c"}" = true; } will be a parse-time error, while { a = {}; "${"a"}".b = null; c = true; } will be an eval-time error (technically that case could theoretically be detected at parse time, but the general case would require full evaluation). Moreover, duplicate dynamic attributes are not allowed even in cases where they would be with static attributes ({ a.b.d = true; a.b.c = false; } is legal, but { a."${"b"}".d = true; a."${"b"}".c = false; } is not). This restriction might be relaxed in the future in cases where the static variant would not be an error, but it is not obvious that that is desirable. Finally, recursive attribute sets with dynamic attributes have the static attributes in scope but not the dynamic ones. So rec { a = true; "${"b"}" = a; } is equivalent to { a = true; b = true; } but rec { "${"a"}" = true; b = a; } would be an error or use a from the surrounding scope if it exists. Note that the getAttr, getAttr or default, and hasAttr are all implemented purely in the parser as syntactic sugar, while attribute sets with dynamic attribute names required changes to the AST to be implemented cleanly. This is an alternative solution to and closes #167 Signed-off-by: Shea Levy <shea@shealevy.com>
2013-09-21 06:25:30 +03:00
2015-07-17 20:24:28 +03:00
i.valueExpr->setName(nameSym);
Dynamic attrs This adds new syntax for attribute names: * attrs."${name}" => getAttr name attrs * attrs ? "${name}" => isAttrs attrs && hasAttr attrs name * attrs."${name}" or def => if attrs ? "${name}" then attrs."${name}" else def * { "${name}" = value; } => listToAttrs [{ inherit name value; }] Of course, it's a bit more complicated than that. The attribute chains can be arbitrarily long and contain combinations of static and dynamic parts (e.g. attrs."${foo}".bar."${baz}" or qux), which is relatively straightforward for the getAttrs/hasAttrs cases but is more complex for the listToAttrs case due to rules about duplicate attribute definitions. For attribute sets with dynamic attribute names, duplicate static attributes are detected at parse time while duplicate dynamic attributes are detected when the attribute set is forced. So, for example, { a = null; a.b = null; "${"c"}" = true; } will be a parse-time error, while { a = {}; "${"a"}".b = null; c = true; } will be an eval-time error (technically that case could theoretically be detected at parse time, but the general case would require full evaluation). Moreover, duplicate dynamic attributes are not allowed even in cases where they would be with static attributes ({ a.b.d = true; a.b.c = false; } is legal, but { a."${"b"}".d = true; a."${"b"}".c = false; } is not). This restriction might be relaxed in the future in cases where the static variant would not be an error, but it is not obvious that that is desirable. Finally, recursive attribute sets with dynamic attributes have the static attributes in scope but not the dynamic ones. So rec { a = true; "${"b"}" = a; } is equivalent to { a = true; b = true; } but rec { "${"a"}" = true; b = a; } would be an error or use a from the surrounding scope if it exists. Note that the getAttr, getAttr or default, and hasAttr are all implemented purely in the parser as syntactic sugar, while attribute sets with dynamic attribute names required changes to the AST to be implemented cleanly. This is an alternative solution to and closes #167 Signed-off-by: Shea Levy <shea@shealevy.com>
2013-09-21 06:25:30 +03:00
/* Keep sorted order so find can catch duplicates */
bindings.insert(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), i.pos);
sort = true;
}
bindings.bindings->pos = pos;
v.mkAttrs(sort ? bindings.finish() : bindings.alreadySorted());
}
void ExprLet::eval(EvalState & state, Env & env, Value & v)
{
/* Create a new environment that contains the attributes in this
`let'. */
Env & env2(state.allocEnv(attrs->attrs.size()));
env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
/* The recursive attributes are evaluated in the new environment,
while the inherited attributes are evaluated in the original
environment. */
2020-02-24 15:33:01 +02:00
Displacement displ = 0;
for (auto & i : attrs->attrs) {
env2.values[displ++] = i.second.e->maybeThunk(
state,
*i.second.chooseByKind(&env2, &env, inheritEnv));
}
auto dts = state.debugRepl
? makeDebugTraceStacker(
state,
*this,
env2,
getPos()
? std::make_shared<Pos>(state.positions[getPos()])
: nullptr,
"while evaluating a '%1%' expression",
"let"
)
: nullptr;
body->eval(state, env2, v);
}
void ExprList::eval(EvalState & state, Env & env, Value & v)
{
auto list = state.buildList(elems.size());
for (const auto & [n, v2] : enumerate(list))
v2 = elems[n]->maybeThunk(state, env);
v.mkList(list);
}
Value * ExprList::maybeThunk(EvalState & state, Env & env)
{
if (elems.empty()) {
return &state.vEmptyList;
}
return Expr::maybeThunk(state, env);
}
void ExprVar::eval(EvalState & state, Env & env, Value & v)
{
2013-10-08 15:24:53 +03:00
Value * v2 = state.lookupVar(&env, *this, false);
state.forceValue(*v2, pos);
v = *v2;
}
static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath)
2015-03-06 15:24:08 +02:00
{
std::ostringstream out;
bool first = true;
for (auto & i : attrPath) {
if (!first) out << '.'; else first = false;
try {
out << state.symbols[getName(i, state, env)];
2015-03-06 15:24:08 +02:00
} catch (Error & e) {
assert(!i.symbol);
out << "\"${";
i.expr->show(state.symbols, out);
out << "}\"";
2015-03-06 15:24:08 +02:00
}
}
return out.str();
}
void ExprSelect::eval(EvalState & state, Env & env, Value & v)
{
Value vTmp;
PosIdx pos2;
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp);
try {
2022-05-23 06:45:24 +03:00
auto dts = state.debugRepl
? makeDebugTraceStacker(
state,
*this,
env,
state.positions[getPos()],
"while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath))
2022-04-09 00:46:12 +03:00
: nullptr;
2013-09-02 17:29:15 +03:00
2015-07-17 20:24:28 +03:00
for (auto & i : attrPath) {
state.nrLookups++;
const Attr * j;
auto name = getName(i, state, env);
if (def) {
state.forceValue(*vAttrs, pos);
if (vAttrs->type() != nAttrs ||
!(j = vAttrs->attrs()->get(name)))
{
def->eval(state, env, v);
return;
}
} else {
state.forceAttrs(*vAttrs, pos, "while selecting an attribute");
if (!(j = vAttrs->attrs()->get(name))) {
std::set<std::string> allAttrNames;
for (auto & attr : *vAttrs->attrs())
allAttrNames.insert(state.symbols[attr.name]);
auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]);
state.error<EvalError>("attribute '%1%' missing", state.symbols[name])
.atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow();
}
}
vAttrs = j->value;
pos2 = j->pos;
if (state.countCalls) state.attrSelects[pos2]++;
}
2013-09-02 17:29:15 +03:00
state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) );
2013-09-02 17:29:15 +03:00
} catch (Error & e) {
if (pos2) {
auto pos2r = state.positions[pos2];
2023-04-06 16:25:06 +03:00
auto origin = std::get_if<SourcePath>(&pos2r.origin);
if (!(origin && *origin == state.derivationInternal))
state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
showAttrPath(state, env, attrPath));
}
throw;
}
2013-09-02 17:29:15 +03:00
v = *vAttrs;
}
2010-04-13 00:21:24 +03:00
void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
{
Value vTmp;
Value * vAttrs = &vTmp;
e->eval(state, env, vTmp);
2015-07-17 20:24:28 +03:00
for (auto & i : attrPath) {
state.forceValue(*vAttrs, getPos());
const Attr * j;
auto name = getName(i, state, env);
if (vAttrs->type() == nAttrs &&
(j = vAttrs->attrs()->get(name)))
{
vAttrs = j->value;
} else {
2022-01-04 19:40:39 +02:00
v.mkBool(false);
return;
}
}
2013-09-02 17:29:15 +03:00
2022-01-04 19:40:39 +02:00
v.mkBool(true);
2010-04-13 00:21:24 +03:00
}
void ExprLambda::eval(EvalState & state, Env & env, Value & v)
{
v.mkLambda(&env, this);
}
namespace {
/** Increments a count on construction and decrements on destruction.
*/
class CallDepth {
size_t & count;
public:
CallDepth(size_t & count) : count(count) {
++count;
}
~CallDepth() {
--count;
}
};
};
void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos)
{
if (callDepth > settings.maxCallDepth)
error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
CallDepth _level(callDepth);
auto trace = settings.traceFunctionCalls
? std::make_unique<FunctionCallTrace>(positions[pos])
: nullptr;
2013-11-07 19:04:36 +02:00
forceValue(fun, pos);
2013-11-07 19:04:36 +02:00
Value vCur(fun);
auto makeAppChain = [&]()
{
vRes = vCur;
for (size_t i = 0; i < nrArgs; ++i) {
auto fun2 = allocValue();
*fun2 = vRes;
vRes.mkPrimOpApp(fun2, args[i]);
}
};
const Attr * functor;
while (nrArgs > 0) {
if (vCur.isLambda()) {
2013-09-02 17:29:15 +03:00
ExprLambda & lambda(*vCur.payload.lambda.fun);
auto size =
(!lambda.arg ? 0 : 1) +
(lambda.hasFormals() ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
env2.up = vCur.payload.lambda.env;
2010-03-30 16:47:59 +03:00
2020-02-24 15:33:01 +02:00
Displacement displ = 0;
if (!lambda.hasFormals())
env2.values[displ++] = args[0];
else {
try {
forceAttrs(*args[0], lambda.pos, "while evaluating the value passed for the lambda argument");
} catch (Error & e) {
if (pos) e.addTrace(positions[pos], "from call site");
throw;
}
2010-03-30 16:47:59 +03:00
if (lambda.arg)
env2.values[displ++] = args[0];
/* For each formal argument, get the actual argument. If
there is no matching actual argument but the formal
argument has a default, use the default. */
size_t attrsUsed = 0;
for (auto & i : lambda.formals->formals) {
auto j = args[0]->attrs()->get(i.name);
if (!j) {
if (!i.def) {
error<TypeError>("function '%1%' called without required argument '%2%'",
(lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
symbols[i.name])
.atPos(lambda.pos)
.withTrace(pos, "from call site")
.withFrame(*fun.payload.lambda.env, lambda)
.debugThrow();
}
env2.values[displ++] = i.def->maybeThunk(*this, env2);
} else {
attrsUsed++;
env2.values[displ++] = j->value;
}
}
/* Check that each actual argument is listed as a formal
argument (unless the attribute match specifies a `...'). */
if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs()->size()) {
/* Nope, so show the first unexpected argument to the
user. */
for (auto & i : *args[0]->attrs())
if (!lambda.formals->has(i.name)) {
std::set<std::string> formalNames;
for (auto & formal : lambda.formals->formals)
formalNames.insert(symbols[formal.name]);
auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]);
error<TypeError>("function '%1%' called with unexpected argument '%2%'",
(lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"),
symbols[i.name])
.atPos(lambda.pos)
.withTrace(pos, "from call site")
.withSuggestions(suggestions)
.withFrame(*fun.payload.lambda.env, lambda)
.debugThrow();
}
abort(); // can't happen
}
}
nrFunctionCalls++;
if (countCalls) incrFunctionCall(&lambda);
/* Evaluate the body. */
try {
2022-05-23 06:45:24 +03:00
auto dts = debugRepl
? makeDebugTraceStacker(
*this, *lambda.body, env2, positions[lambda.pos],
"while calling %s",
lambda.name
? concatStrings("'", symbols[lambda.name], "'")
: "anonymous lambda")
: nullptr;
2021-12-23 04:40:08 +02:00
lambda.body->eval(*this, env2, vCur);
} catch (Error & e) {
if (loggerSettings.showTrace.get()) {
addErrorTrace(
e,
lambda.pos,
"while calling %s",
lambda.name
? concatStrings("'", symbols[lambda.name], "'")
2024-02-23 03:14:55 +02:00
: "anonymous lambda");
2024-02-23 03:18:27 +02:00
if (pos) addErrorTrace(e, pos, "from call site");
}
throw;
}
nrArgs--;
args += 1;
}
else if (vCur.isPrimOp()) {
size_t argsLeft = vCur.primOp()->arity;
if (nrArgs < argsLeft) {
/* We don't have enough arguments, so create a tPrimOpApp chain. */
makeAppChain();
return;
2021-08-25 22:18:27 +03:00
} else {
/* We have all the arguments, so call the primop. */
auto * fn = vCur.primOp();
nrPrimOpCalls++;
if (countCalls) primOpCalls[fn->name]++;
try {
fn->fun(*this, vCur.determinePos(noPos), args, vCur);
} catch (Error & e) {
if (fn->addTrace)
addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
throw;
}
nrArgs -= argsLeft;
args += argsLeft;
2021-08-06 20:09:27 +03:00
}
}
2010-03-30 16:47:59 +03:00
else if (vCur.isPrimOpApp()) {
/* Figure out the number of arguments still needed. */
size_t argsDone = 0;
Value * primOp = &vCur;
while (primOp->isPrimOpApp()) {
argsDone++;
primOp = primOp->payload.primOpApp.left;
}
assert(primOp->isPrimOp());
auto arity = primOp->primOp()->arity;
auto argsLeft = arity - argsDone;
if (nrArgs < argsLeft) {
/* We still don't have enough arguments, so extend the tPrimOpApp chain. */
makeAppChain();
return;
} else {
/* We have all the arguments, so call the primop with
the previous and new arguments. */
Value * vArgs[maxPrimOpArity];
auto n = argsDone;
for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->payload.primOpApp.left)
vArgs[--n] = arg->payload.primOpApp.right;
for (size_t i = 0; i < argsLeft; ++i)
vArgs[argsDone + i] = args[i];
auto fn = primOp->primOp();
nrPrimOpCalls++;
if (countCalls) primOpCalls[fn->name]++;
try {
// TODO:
// 1. Unify this and above code. Heavily redundant.
// 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc)
// so the debugger allows to inspect the wrong parameters passed to the builtin.
fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
} catch (Error & e) {
if (fn->addTrace)
addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
throw;
}
nrArgs -= argsLeft;
args += argsLeft;
}
}
2021-08-25 22:18:27 +03:00
else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(sFunctor))) {
/* 'vCur' may be allocated on the stack of the calling
function, but for functors we may keep a reference, so
heap-allocate a copy and use that instead. */
Value * args2[] = {allocValue(), args[0]};
*args2[0] = vCur;
try {
callFunction(*functor->value, 2, args2, vCur, functor->pos);
} catch (Error & e) {
e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)");
throw;
}
nrArgs--;
args++;
}
else
error<TypeError>(
"attempt to call something which is not a function but %1%: %2%",
showType(vCur),
ValuePrinter(*this, vCur, errorPrintOptions))
.atPos(pos)
.debugThrow();
2010-03-30 16:47:59 +03:00
}
vRes = vCur;
}
void ExprCall::eval(EvalState & state, Env & env, Value & v)
{
auto dts = state.debugRepl
? makeDebugTraceStacker(
state,
*this,
env,
getPos()
? std::make_shared<Pos>(state.positions[getPos()])
: nullptr,
"while calling a function"
)
: nullptr;
Value vFun;
fun->eval(state, env, vFun);
// Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5}
// 2: over 4000
// 3: about 300
// 4: about 60
// 5: under 10
// This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
SmallValueVector<4> vArgs(args.size());
for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, args.size(), vArgs.data(), v, pos);
2013-11-07 19:04:36 +02:00
}
// Lifted out of callFunction() because it creates a temporary that
// prevents tail-call optimisation.
void EvalState::incrFunctionCall(ExprLambda * fun)
{
functionCalls[fun]++;
2010-03-30 16:47:59 +03:00
}
void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
{
2022-02-04 01:31:33 +02:00
auto pos = fun.determinePos(noPos);
forceValue(fun, pos);
if (fun.type() == nAttrs) {
auto found = fun.attrs()->find(sFunctor);
if (found != fun.attrs()->end()) {
2015-11-25 18:56:14 +02:00
Value * v = allocValue();
callFunction(*found->value, fun, *v, pos);
forceValue(*v, pos);
2015-11-25 18:56:14 +02:00
return autoCallFunction(args, *v, res);
}
}
if (!fun.isLambda() || !fun.payload.lambda.fun->hasFormals()) {
res = fun;
return;
}
auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.payload.lambda.fun->formals->formals.size()), args.size()));
if (fun.payload.lambda.fun->formals->ellipsis) {
// If the formals have an ellipsis (eg the function accepts extra args) pass
// all available automatic arguments (which includes arguments specified on
// the command line via --arg/--argstr)
for (auto & v : args)
attrs.insert(v);
} else {
// Otherwise, only pass the arguments that the function accepts
for (auto & i : fun.payload.lambda.fun->formals->formals) {
auto j = args.get(i.name);
if (j) {
attrs.insert(*j);
} else if (!i.def) {
error<MissingArgumentError>(R"(cannot evaluate a function that has an argument without a value ('%1%')
2021-02-22 16:24:14 +02:00
Nix attempted to evaluate a function as a top level expression; in
this case it must have its arguments supplied either by default
values, or passed explicitly with '--arg' or '--argstr'. See
https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name])
.atPos(i.pos).withFrame(*fun.payload.lambda.env, *fun.payload.lambda.fun).debugThrow();
}
}
}
callFunction(fun, allocValue()->mkAttrs(attrs), res, pos);
}
void ExprWith::eval(EvalState & state, Env & env, Value & v)
{
2010-04-14 18:01:04 +03:00
Env & env2(state.allocEnv(1));
env2.up = &env;
env2.values[0] = attrs->maybeThunk(state, env);
2021-11-09 22:14:49 +02:00
body->eval(state, env2, v);
}
void ExprIf::eval(EvalState & state, Env & env, Value & v)
{
// We cheat in the parser, and pass the position of the condition as the position of the if itself.
(state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v);
}
2013-09-02 17:29:15 +03:00
2010-04-13 00:21:24 +03:00
void ExprAssert::eval(EvalState & state, Env & env, Value & v)
{
if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) {
2024-07-05 17:43:48 +03:00
auto exprStr = ({
std::ostringstream out;
cond->show(state.symbols, out);
out.str();
});
if (auto eq = dynamic_cast<ExprOpEq *>(cond)) {
try {
Value v1; eq->e1->eval(state, env, v1);
Value v2; eq->e2->eval(state, env, v2);
state.assertEqValues(v1, v2, eq->pos, "in an equality assertion");
} catch (AssertionError & e) {
e.addTrace(state.positions[pos], "while evaluating the condition of the assertion '%s'", exprStr);
throw;
}
}
state.error<AssertionError>("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow();
}
body->eval(state, env, v);
2010-04-13 00:21:24 +03:00
}
2013-09-02 17:29:15 +03:00
2010-04-13 00:21:24 +03:00
void ExprOpNot::eval(EvalState & state, Env & env, Value & v)
{
v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: !
2010-04-13 00:21:24 +03:00
}
void ExprOpEq::eval(EvalState & state, Env & env, Value & v)
{
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
v.mkBool(state.eqValues(v1, v2, pos, "while testing two values for equality"));
}
void ExprOpNEq::eval(EvalState & state, Env & env, Value & v)
{
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
v.mkBool(!state.eqValues(v1, v2, pos, "while testing two values for inequality"));
}
void ExprOpAnd::eval(EvalState & state, Env & env, Value & v)
{
v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator"));
}
void ExprOpOr::eval(EvalState & state, Env & env, Value & v)
{
v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator"));
}
void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
{
v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator"));
}
void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
{
Value v1, v2;
state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator");
state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator");
2010-10-20 18:48:00 +03:00
state.nrOpUpdates++;
if (v1.attrs()->size() == 0) { v = v2; return; }
if (v2.attrs()->size() == 0) { v = v1; return; }
auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size());
/* Merge the sets, preferring values from the second set. Make
sure to keep the resulting vector in sorted order. */
auto i = v1.attrs()->begin();
auto j = v2.attrs()->begin();
while (i != v1.attrs()->end() && j != v2.attrs()->end()) {
if (i->name == j->name) {
2022-01-04 21:29:17 +02:00
attrs.insert(*j);
++i; ++j;
}
else if (i->name < j->name)
2022-01-04 21:29:17 +02:00
attrs.insert(*i++);
else
2022-01-04 21:29:17 +02:00
attrs.insert(*j++);
}
2010-10-20 18:48:00 +03:00
while (i != v1.attrs()->end()) attrs.insert(*i++);
while (j != v2.attrs()->end()) attrs.insert(*j++);
2022-01-04 21:29:17 +02:00
v.mkAttrs(attrs.alreadySorted());
2013-09-02 17:29:15 +03:00
state.nrOpUpdateValuesCopied += v.attrs()->size();
}
void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v)
{
Value v1; e1->eval(state, env, v1);
Value v2; e2->eval(state, env, v2);
Value * lists[2] = { &v1, &v2 };
state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate");
}
void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, const PosIdx pos, std::string_view errorCtx)
{
nrListConcats++;
Value * nonEmpty = 0;
2018-05-02 14:56:34 +03:00
size_t len = 0;
for (size_t n = 0; n < nrLists; ++n) {
forceList(*lists[n], pos, errorCtx);
2018-05-02 14:56:34 +03:00
auto l = lists[n]->listSize();
len += l;
if (l) nonEmpty = lists[n];
}
if (nonEmpty && len == nonEmpty->listSize()) {
v = *nonEmpty;
return;
}
auto list = buildList(len);
auto out = list.elems;
2018-05-02 14:56:34 +03:00
for (size_t n = 0, pos = 0; n < nrLists; ++n) {
auto l = lists[n]->listSize();
if (l)
memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *));
pos += l;
}
v.mkList(list);
}
2010-04-13 00:21:24 +03:00
void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
{
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
NixStringContext context;
std::vector<BackedStringView> s;
size_t sSize = 0;
NixInt n = 0;
NixFloat nf = 0;
bool first = !forceString;
ValueType firstType = nString;
2010-04-13 00:21:24 +03:00
const auto str = [&] {
std::string result;
result.reserve(sSize);
for (const auto & part : s) result += *part;
return result;
};
/* c_str() is not str().c_str() because we want to create a string
Value. allocating a GC'd string directly and moving it into a
Value lets us avoid an allocation and copy. */
const auto c_str = [&] {
char * result = allocString(sSize + 1);
char * tmp = result;
for (const auto & part : s) {
memcpy(tmp, part->data(), part->size());
tmp += part->size();
}
*tmp = 0;
return result;
};
// List of returned strings. References to these Values must NOT be persisted.
SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
Value * vTmpP = values.data();
for (auto & [i_pos, i] : *es) {
Value & vTmp = *vTmpP++;
2015-07-17 20:24:28 +03:00
i->eval(state, env, vTmp);
2010-04-13 00:21:24 +03:00
/* If the first element is a path, then the result will also
be a path, we don't copy anything (yet - that's done later,
since paths are copied when they are used in a derivation),
and none of the strings are allowed to have contexts. */
if (first) {
firstType = vTmp.type();
2010-04-13 00:21:24 +03:00
}
if (firstType == nInt) {
if (vTmp.type() == nInt) {
n += vTmp.integer();
} else if (vTmp.type() == nFloat) {
// Upgrade the type from int to float;
firstType = nFloat;
nf = n;
nf += vTmp.fpoint();
} else
state.error<EvalError>("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
} else if (firstType == nFloat) {
if (vTmp.type() == nInt) {
nf += vTmp.integer();
} else if (vTmp.type() == nFloat) {
nf += vTmp.fpoint();
} else
state.error<EvalError>("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow();
} else {
if (s.empty()) s.reserve(es->size());
2021-07-29 19:03:07 +03:00
/* skip canonization of first path, which would only be not
canonized in the first place if it's coming from a ./${foo} type
path */
2022-11-29 01:25:36 +02:00
auto part = state.coerceToString(i_pos, vTmp, context,
"while evaluating a path segment",
false, firstType == nString, !first);
sSize += part->size();
s.emplace_back(std::move(part));
}
2021-07-29 19:03:07 +03:00
first = false;
2010-04-13 00:21:24 +03:00
}
if (firstType == nInt)
2022-01-04 19:40:39 +02:00
v.mkInt(n);
else if (firstType == nFloat)
2022-01-04 19:40:39 +02:00
v.mkFloat(nf);
else if (firstType == nPath) {
if (!context.empty())
state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else
v.mkStringMove(c_str(), context);
2010-04-13 00:21:24 +03:00
}
void ExprPos::eval(EvalState & state, Env & env, Value & v)
{
state.mkPos(v, pos);
}
void ExprBlackHole::eval(EvalState & state, Env & env, Value & v)
{
state.error<InfiniteRecursionError>("infinite recursion encountered")
.atPos(v.determinePos(noPos))
.debugThrow();
}
// always force this to be separate, otherwise forceValue may inline it and take
// a massive perf hit
[[gnu::noinline]]
void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos)
{
if (!v.isBlackhole())
return;
auto e = std::current_exception();
try {
std::rethrow_exception(e);
} catch (InfiniteRecursionError & e) {
e.atPos(positions[pos]);
} catch (...) {
}
}
void EvalState::forceValueDeep(Value & v)
{
std::set<const Value *> seen;
2013-09-02 17:29:15 +03:00
std::function<void(Value & v)> recurse;
2013-09-02 17:29:15 +03:00
recurse = [&](Value & v) {
if (!seen.insert(&v).second) return;
forceValue(v, v.determinePos(noPos));
if (v.type() == nAttrs) {
for (auto & i : *v.attrs())
2015-03-06 16:10:12 +02:00
try {
// If the value is a thunk, we're evaling. Otherwise no trace necessary.
2022-05-23 06:45:24 +03:00
auto dts = debugRepl && i.value->isThunk()
? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos],
"while evaluating the attribute '%1%'", symbols[i.name])
2022-04-09 00:46:12 +03:00
: nullptr;
2021-12-23 04:40:08 +02:00
2015-03-06 16:10:12 +02:00
recurse(*i.value);
} catch (Error & e) {
addErrorTrace(e, i.pos, "while evaluating the attribute '%1%'", symbols[i.name]);
2015-03-06 16:10:12 +02:00
throw;
}
}
else if (v.isList()) {
for (auto v2 : v.listItems())
recurse(*v2);
}
};
recurse(v);
}
NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCtx)
{
try {
forceValue(v, pos);
if (v.type() != nInt)
error<TypeError>(
"expected an integer but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.integer();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
return v.integer();
}
NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx)
{
try {
forceValue(v, pos);
if (v.type() == nInt)
return v.integer();
else if (v.type() != nFloat)
error<TypeError>(
"expected a float but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.fpoint();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
}
bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx)
2010-03-31 18:38:03 +03:00
{
try {
forceValue(v, pos);
if (v.type() != nBool)
error<TypeError>(
"expected a Boolean but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.boolean();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
return v.boolean();
2010-03-31 18:38:03 +03:00
}
2015-10-08 14:22:11 +03:00
bool EvalState::isFunctor(Value & fun)
{
return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end();
2015-10-08 14:22:11 +03:00
}
void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx)
2010-03-30 16:47:59 +03:00
{
try {
forceValue(v, pos);
if (v.type() != nFunction && !isFunctor(v))
error<TypeError>(
"expected a function but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
2010-03-30 16:47:59 +03:00
}
std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string_view errorCtx)
2010-03-30 21:05:54 +03:00
{
try {
forceValue(v, pos);
if (v.type() != nString)
error<TypeError>(
"expected a string but found %1%: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
).atPos(pos).debugThrow();
return v.string_view();
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
2014-04-04 22:14:11 +03:00
}
2010-03-31 18:38:03 +03:00
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
void copyContext(const Value & v, NixStringContext & context)
{
if (v.payload.string.context)
for (const char * * p = v.payload.string.context; *p; ++p)
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
context.insert(NixStringContextElem::parse(*p));
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx)
{
auto s = forceString(v, pos, errorCtx);
copyContext(v, context);
return s;
}
std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx)
2010-03-31 18:38:03 +03:00
{
auto s = forceString(v, pos, errorCtx);
if (v.context()) {
error<EvalError>("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow();
2014-04-04 22:14:11 +03:00
}
2010-03-31 18:38:03 +03:00
return s;
2010-03-30 21:05:54 +03:00
}
bool EvalState::isDerivation(Value & v)
{
if (v.type() != nAttrs) return false;
auto i = v.attrs()->get(sType);
if (!i) return false;
forceValue(*i->value, i->pos);
if (i->value->type() != nString) return false;
return i->value->string_view().compare("derivation") == 0;
}
std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v,
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
NixStringContext & context, bool coerceMore, bool copyToStore)
2019-10-27 11:15:51 +02:00
{
auto i = v.attrs()->find(sToString);
if (i != v.attrs()->end()) {
2019-10-27 11:15:51 +02:00
Value v1;
callFunction(*i->value, v, v1, pos);
2022-11-29 01:25:36 +02:00
return coerceToString(pos, v1, context,
"while evaluating the result of the `__toString` attribute",
2022-11-29 01:25:36 +02:00
coerceMore, copyToStore).toOwned();
2019-10-27 11:15:51 +02:00
}
return {};
}
BackedStringView EvalState::coerceToString(
const PosIdx pos,
Value & v,
NixStringContext & context,
std::string_view errorCtx,
bool coerceMore,
bool copyToStore,
bool canonicalizePath)
2010-03-30 12:22:33 +03:00
{
forceValue(v, pos);
2010-03-30 12:22:33 +03:00
if (v.type() == nString) {
copyContext(v, context);
return v.string_view();
}
2010-03-30 12:22:33 +03:00
if (v.type() == nPath) {
return
!canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}.
v.payload.path.path
: copyToStore
? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs());
2010-03-30 12:22:33 +03:00
}
if (v.type() == nAttrs) {
2019-10-27 11:15:51 +02:00
auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
if (maybeString)
return std::move(*maybeString);
auto i = v.attrs()->find(sOutPath);
if (i == v.attrs()->end()) {
error<TypeError>(
"cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
)
2022-11-29 01:25:36 +02:00
.withTrace(pos, errorCtx)
.debugThrow();
2022-11-29 01:25:36 +02:00
}
return coerceToString(pos, *i->value, context, errorCtx,
coerceMore, copyToStore, canonicalizePath);
2010-03-30 12:22:33 +03:00
}
2022-11-29 01:25:36 +02:00
if (v.type() == nExternal) {
try {
return v.external()->coerceToString(*this, pos, context, coerceMore, copyToStore);
2022-11-29 01:25:36 +02:00
} catch (Error & e) {
e.addTrace(nullptr, errorCtx);
throw;
}
}
2010-03-30 12:22:33 +03:00
if (coerceMore) {
/* Note that `false' is represented as an empty string for
shell scripting convenience, just like `null'. */
if (v.type() == nBool && v.boolean()) return "1";
if (v.type() == nBool && !v.boolean()) return "";
if (v.type() == nInt) return std::to_string(v.integer());
if (v.type() == nFloat) return std::to_string(v.fpoint());
if (v.type() == nNull) return "";
2010-03-30 12:22:33 +03:00
if (v.isList()) {
std::string result;
for (auto [n, v2] : enumerate(v.listItems())) {
try {
result += *coerceToString(pos, *v2, context,
2022-11-29 01:25:36 +02:00
"while evaluating one element of the list",
coerceMore, copyToStore, canonicalizePath);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
if (n < v.listSize() - 1
/* !!! not quite correct */
&& (!v2->isList() || v2->listSize() != 0))
result += " ";
2010-03-30 12:22:33 +03:00
}
return result;
2010-03-30 12:22:33 +03:00
}
}
2013-09-02 17:29:15 +03:00
error<TypeError>("cannot coerce %1% to a string: %2%",
showType(v),
ValuePrinter(*this, v, errorPrintOptions)
)
2022-11-29 01:25:36 +02:00
.withTrace(pos, errorCtx)
.debugThrow();
2010-03-30 12:22:33 +03:00
}
StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path)
2013-11-19 01:03:11 +02:00
{
if (nix::isDerivation(path.path.abs()))
error<EvalError>("file names are not allowed to end in '%1%'", drvExtension).debugThrow();
2013-11-19 01:03:11 +02:00
2024-06-04 17:05:56 +03:00
auto dstPathCached = get(*srcToStore.lock(), path);
2024-06-04 17:05:56 +03:00
auto dstPath = dstPathCached
? *dstPathCached
: [&]() {
auto dstPath = fetchToStore(
*store,
path.resolveSymlinks(),
settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
path.baseName(),
ContentAddressMethod::Raw::NixArchive,
nullptr,
repair);
allowPath(dstPath);
2024-06-04 17:05:56 +03:00
srcToStore.lock()->try_emplace(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
return dstPath;
}();
2013-11-19 01:03:11 +02:00
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
context.insert(NixStringContextElem::Opaque {
.path = dstPath
});
2013-11-19 01:03:11 +02:00
return dstPath;
}
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
2010-03-30 12:22:33 +03:00
{
try {
forceValue(v, pos);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
/* Handle path values directly, without coercing to a string. */
if (v.type() == nPath)
return v.path();
/* Similarly, handle __toString where the result may be a path
value. */
if (v.type() == nAttrs) {
auto i = v.attrs()->find(sToString);
if (i != v.attrs()->end()) {
Value v1;
callFunction(*i->value, v, v1, pos);
return coerceToPath(pos, v1, context, errorCtx);
}
}
/* Any other value should be coercable to a string, interpreted
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/')
error<EvalError>("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow();
return rootPath(CanonPath(path));
2010-03-30 12:22:33 +03:00
}
Use `std::set<StringContextElem>` not `PathSet` for string contexts Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std<StringContextElem>`. ----- In support of this change, `NixStringContext` is now defined as `std::std<StringContextElem>` not `std:vector<StringContextElem>`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector<StringContextElem>` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set<StringContextElem>` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com>
2023-01-29 03:31:10 +02:00
StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{
2022-11-29 01:25:36 +02:00
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (auto storePath = store->maybeParseStorePath(path))
return *storePath;
error<EvalError>("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow();
}
std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
{
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx);
auto csize = context.size();
if (csize != 1)
error<EvalError>(
"string '%s' has %d entries in its context. It should only have exactly one entry",
s, csize)
.withTrace(pos, errorCtx).debugThrow();
auto derivedPath = std::visit(overloaded {
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath {
return std::move(o);
},
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error<EvalError>(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s).withTrace(pos, errorCtx).debugThrow();
},
[&](NixStringContextElem::Built && b) -> SingleDerivedPath {
return std::move(b);
},
}, ((NixStringContextElem &&) *context.begin()).raw);
return {
std::move(derivedPath),
std::move(s),
};
}
SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
auto sExpected = mkSingleDerivedPathStringRaw(derivedPath);
if (s != sExpected) {
/* `std::visit` is used here just to provide a more precise
error message. */
std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
error<EvalError>(
"path string '%s' has context with the different path '%s'",
s, sExpected)
.withTrace(pos, errorCtx).debugThrow();
},
[&](const SingleDerivedPath::Built & b) {
error<EvalError>(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, b.output, b.drvPath->to_string(*store), sExpected)
.withTrace(pos, errorCtx).debugThrow();
}
}, derivedPath.raw());
}
return derivedPath;
}
2024-07-05 17:43:48 +03:00
// NOTE: This implementation must match eqValues!
// We accept this burden because informative error messages for
// `assert a == b; x` are critical for our users' testing UX.
void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{
// This implementation must match eqValues.
forceValue(v1, pos);
forceValue(v2, pos);
if (&v1 == &v2)
return;
// Special case type-compatibility between float and int
if ((v1.type() == nInt || v1.type() == nFloat) && (v2.type() == nInt || v2.type() == nFloat)) {
if (eqValues(v1, v2, pos, errorCtx)) {
return;
} else {
error<AssertionError>(
"%s with value '%s' is not equal to %s with value '%s'",
showType(v1),
ValuePrinter(*this, v1, errorPrintOptions),
showType(v2),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
}
if (v1.type() != v2.type()) {
error<AssertionError>(
"%s of value '%s' is not equal to %s of value '%s'",
showType(v1),
ValuePrinter(*this, v1, errorPrintOptions),
showType(v2),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
switch (v1.type()) {
case nInt:
if (v1.integer() != v2.integer()) {
error<AssertionError>("integer '%d' is not equal to integer '%d'", v1.integer(), v2.integer()).debugThrow();
}
return;
case nBool:
if (v1.boolean() != v2.boolean()) {
error<AssertionError>(
"boolean '%s' is not equal to boolean '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
return;
case nString:
if (strcmp(v1.c_str(), v2.c_str()) != 0) {
error<AssertionError>(
"string '%s' is not equal to string '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
return;
case nPath:
if (v1.payload.path.accessor != v2.payload.path.accessor) {
error<AssertionError>(
"path '%s' is not equal to path '%s' because their accessors are different",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) {
error<AssertionError>(
"path '%s' is not equal to path '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
return;
case nNull:
return;
case nList:
if (v1.listSize() != v2.listSize()) {
error<AssertionError>(
"list of size '%d' is not equal to list of size '%d', left hand side is '%s', right hand side is '%s'",
v1.listSize(),
v2.listSize(),
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
for (size_t n = 0; n < v1.listSize(); ++n) {
try {
assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx);
} catch (Error & e) {
e.addTrace(positions[pos], "while comparing list element %d", n);
throw;
}
}
return;
case nAttrs: {
if (isDerivation(v1) && isDerivation(v2)) {
auto i = v1.attrs()->get(sOutPath);
auto j = v2.attrs()->get(sOutPath);
if (i && j) {
try {
assertEqValues(*i->value, *j->value, pos, errorCtx);
return;
} catch (Error & e) {
e.addTrace(positions[pos], "while comparing a derivation by its '%s' attribute", "outPath");
throw;
}
assert(false);
}
}
if (v1.attrs()->size() != v2.attrs()->size()) {
error<AssertionError>(
"attribute names of attribute set '%s' differs from attribute set '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
// Like normal comparison, we compare the attributes in non-deterministic Symbol index order.
// This function is called when eqValues has found a difference, so to reliably
// report about its result, we should follow in its literal footsteps and not
// try anything fancy that could lead to an error.
Bindings::const_iterator i, j;
for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) {
if (i->name != j->name) {
// A difference in a sorted list means that one attribute is not contained in the other, but we don't
// know which. Let's find out. Could use <, but this is more clear.
if (!v2.attrs()->get(i->name)) {
error<AssertionError>(
"attribute name '%s' is contained in '%s', but not in '%s'",
symbols[i->name],
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
if (!v1.attrs()->get(j->name)) {
error<AssertionError>(
"attribute name '%s' is missing in '%s', but is contained in '%s'",
symbols[j->name],
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
assert(false);
}
try {
assertEqValues(*i->value, *j->value, pos, errorCtx);
} catch (Error & e) {
// The order of traces is reversed, so this presents as
// where left hand side is
// at <pos>
// where right hand side is
// at <pos>
// while comparing attribute '<name>'
if (j->pos != noPos)
e.addTrace(positions[j->pos], "where right hand side is");
if (i->pos != noPos)
e.addTrace(positions[i->pos], "where left hand side is");
e.addTrace(positions[pos], "while comparing attribute '%s'", symbols[i->name]);
throw;
}
}
return;
}
case nFunction:
error<AssertionError>("distinct functions and immediate comparisons of identical functions compare as unequal")
.debugThrow();
case nExternal:
if (!(*v1.external() == *v2.external())) {
error<AssertionError>(
"external value '%s' is not equal to external value '%s'",
ValuePrinter(*this, v1, errorPrintOptions),
ValuePrinter(*this, v2, errorPrintOptions))
.debugThrow();
}
return;
case nFloat:
// !!!
if (!(v1.fpoint() == v2.fpoint())) {
error<AssertionError>("float '%f' is not equal to float '%f'", v1.fpoint(), v2.fpoint()).debugThrow();
}
return;
case nThunk: // Must not be left by forceValue
default:
// This should never happen, because eqValues already throws an
// error for this, and this function should only be called when
// eqValues has found a difference, and it should match
// its behavior.
error<EvalBaseError>(
"cannot compare %1% with %2%; is assertEqValues out of sync with eqValues?", showType(v1), showType(v2))
.debugThrow();
}
}
// This implementation must match assertEqValues
bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx)
{
forceValue(v1, pos);
forceValue(v2, pos);
/* !!! Hack to support some old broken code that relies on pointer
equality tests between sets. (Specifically, builderDefs calls
uniqList on a list of sets.) Will remove this eventually. */
if (&v1 == &v2) return true;
// Special case type-compatibility between float and int
if (v1.type() == nInt && v2.type() == nFloat)
return v1.integer() == v2.fpoint();
if (v1.type() == nFloat && v2.type() == nInt)
return v1.fpoint() == v2.integer();
// All other types are not compatible with each other.
if (v1.type() != v2.type()) return false;
switch (v1.type()) {
case nInt:
return v1.integer() == v2.integer();
case nBool:
return v1.boolean() == v2.boolean();
case nString:
return strcmp(v1.c_str(), v2.c_str()) == 0;
case nPath:
return
2023-10-20 17:58:33 +03:00
// FIXME: compare accessors by their fingerprint.
v1.payload.path.accessor == v2.payload.path.accessor
&& strcmp(v1.payload.path.path, v2.payload.path.path) == 0;
case nNull:
2010-03-31 12:54:12 +03:00
return true;
case nList:
if (v1.listSize() != v2.listSize()) return false;
2018-05-02 14:56:34 +03:00
for (size_t n = 0; n < v1.listSize(); ++n)
if (!eqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx)) return false;
return true;
case nAttrs: {
/* If both sets denote a derivation (type = "derivation"),
then compare their outPaths. */
if (isDerivation(v1) && isDerivation(v2)) {
auto i = v1.attrs()->get(sOutPath);
auto j = v2.attrs()->get(sOutPath);
if (i && j)
return eqValues(*i->value, *j->value, pos, errorCtx);
}
if (v1.attrs()->size() != v2.attrs()->size()) return false;
/* Otherwise, compare the attributes one by one. */
Bindings::const_iterator i, j;
for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j)
if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx))
return false;
2013-09-02 17:29:15 +03:00
return true;
}
2010-04-01 13:55:36 +03:00
/* Functions are incomparable. */
case nFunction:
2010-04-01 13:55:36 +03:00
return false;
case nExternal:
return *v1.external() == *v2.external();
case nFloat:
2024-07-05 17:43:48 +03:00
// !!!
return v1.fpoint() == v2.fpoint();
case nThunk: // Must not be left by forceValue
default:
error<EvalError>("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow();
}
}
bool EvalState::fullGC() {
#if HAVE_BOEHMGC
GC_gcollect();
// Check that it ran. We might replace this with a version that uses more
// of the boehm API to get this reliably, at a maintenance cost.
// We use a 1K margin because technically this has a race condtion, but we
// probably won't encounter it in practice, because the CLI isn't concurrent
// like that.
return GC_get_bytes_since_gc() < 1024;
#else
return false;
#endif
}
2023-10-09 17:25:53 +03:00
void EvalState::maybePrintStats()
{
bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0";
2012-02-04 15:27:11 +02:00
if (showStats) {
// Make the final heap size more deterministic.
#if HAVE_BOEHMGC
if (!fullGC()) {
warn("failed to perform a full GC before reporting stats");
}
#endif
printStatistics();
}
}
void EvalState::printStatistics()
{
#ifndef _WIN32 // TODO use portable implementation
2012-02-04 15:27:11 +02:00
struct rusage buf;
getrusage(RUSAGE_SELF, &buf);
float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000);
#endif
2012-02-04 15:27:11 +02:00
2014-10-05 01:39:28 +03:00
uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *);
uint64_t bLists = nrListElems * sizeof(Value *);
uint64_t bValues = nrValues * sizeof(Value);
uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr);
2015-03-18 17:24:54 +02:00
#if HAVE_BOEHMGC
GC_word heapSize, totalBytes;
GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes);
#endif
auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-");
std::fstream fs;
if (outPath != "-")
fs.open(outPath, std::fstream::out);
json topObj = json::object();
#ifndef _WIN32 // TODO implement
topObj["cpuTime"] = cpuTime;
#endif
topObj["envs"] = {
{"number", nrEnvs},
{"elements", nrValuesInEnvs},
{"bytes", bEnvs},
};
topObj["nrExprs"] = Expr::nrExprs;
topObj["list"] = {
{"elements", nrListElems},
{"bytes", bLists},
{"concats", nrListConcats},
};
topObj["values"] = {
{"number", nrValues},
{"bytes", bValues},
};
topObj["symbols"] = {
{"number", symbols.size()},
{"bytes", symbols.totalSize()},
};
topObj["sets"] = {
{"number", nrAttrsets},
{"bytes", bAttrsets},
{"elements", nrAttrsInAttrsets},
};
topObj["sizes"] = {
{"Env", sizeof(Env)},
{"Value", sizeof(Value)},
{"Bindings", sizeof(Bindings)},
{"Attr", sizeof(Attr)},
};
topObj["nrOpUpdates"] = nrOpUpdates;
topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied;
topObj["nrThunks"] = nrThunks;
topObj["nrAvoided"] = nrAvoided;
topObj["nrLookups"] = nrLookups;
topObj["nrPrimOpCalls"] = nrPrimOpCalls;
topObj["nrFunctionCalls"] = nrFunctionCalls;
2018-09-02 01:50:22 +03:00
#if HAVE_BOEHMGC
topObj["gc"] = {
{"heapSize", heapSize},
{"totalBytes", totalBytes},
};
2018-09-02 01:50:22 +03:00
#endif
if (countCalls) {
topObj["primops"] = primOpCalls;
{
auto& list = topObj["functions"];
list = json::array();
for (auto & [fun, count] : functionCalls) {
json obj = json::object();
if (fun->name)
obj["name"] = (std::string_view) symbols[fun->name];
else
obj["name"] = nullptr;
if (auto pos = positions[fun->pos]) {
if (auto path = std::get_if<SourcePath>(&pos.origin))
obj["file"] = path->to_string();
obj["line"] = pos.line;
obj["column"] = pos.column;
2018-09-05 22:57:54 +03:00
}
obj["count"] = count;
list.push_back(obj);
2018-09-05 22:57:54 +03:00
}
}
{
auto list = topObj["attributes"];
list = json::array();
for (auto & i : attrSelects) {
json obj = json::object();
if (auto pos = positions[i.first]) {
if (auto path = std::get_if<SourcePath>(&pos.origin))
obj["file"] = path->to_string();
obj["line"] = pos.line;
obj["column"] = pos.column;
2018-09-05 22:57:54 +03:00
}
obj["count"] = i.second;
list.push_back(obj);
2018-09-05 22:57:54 +03:00
}
}
}
if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") {
// XXX: overrides earlier assignment
topObj["symbols"] = json::array();
auto &list = topObj["symbols"];
symbols.dump([&](const std::string & s) { list.emplace_back(s); });
}
if (outPath == "-") {
std::cerr << topObj.dump(2) << std::endl;
} else {
fs << topObj.dump(2) << std::endl;
}
}
SourcePath resolveExprPath(SourcePath path)
{
unsigned int followCount = 0, maxFollow = 1024;
/* If `path' is a symlink, follow it. This is so that relative
path references work. */
while (!path.path.isRoot()) {
// Basic cycle/depth limit to avoid infinite loops.
if (++followCount >= maxFollow)
throw Error("too many symbolic links encountered while traversing the path '%s'", path);
auto p = path.parent().resolveSymlinks() / path.baseName();
if (p.lstat().type != SourceAccessor::tSymlink) break;
path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))};
}
/* If `path' refers to a directory, append `/default.nix'. */
if (path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory)
return path / "default.nix";
return path;
}
Expr * EvalState::parseExprFromFile(const SourcePath & path)
{
return parseExprFromFile(path, staticBaseEnv);
}
Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
{
auto buffer = path.resolveSymlinks().readFile();
// readFile hopefully have left some extra space for terminators
buffer.append("\0\0", 2);
return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
}
Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
{
// NOTE this method (and parseStdin) must take care to *fully copy* their input
// into their respective Pos::Origin until the parser stops overwriting its input
// data.
auto s = make_ref<std::string>(s_);
s_.append("\0\0", 2);
return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv);
}
Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath)
{
return parseExprFromString(std::move(s), basePath, staticBaseEnv);
}
Expr * EvalState::parseStdin()
{
// NOTE this method (and parseExprFromString) must take care to *fully copy* their
// input into their respective Pos::Origin until the parser stops overwriting its
// input data.
//Activity act(*logger, lvlTalkative, "parsing standard input");
auto buffer = drainFD(0);
// drainFD should have left some extra space for terminators
buffer.append("\0\0", 2);
auto s = make_ref<std::string>(buffer);
return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv);
}
SourcePath EvalState::findFile(const std::string_view path)
{
return findFile(lookupPath, path);
}
SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_view path, const PosIdx pos)
{
for (auto & i : lookupPath.elements) {
auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
if (!suffixOpt) continue;
auto suffix = *suffixOpt;
auto rOpt = resolveLookupPathPath(i.path);
if (!rOpt) continue;
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
}
if (hasPrefix(path, "nix/"))
return {corepkgsFS, CanonPath(path.substr(3))};
error<ThrownError>(
settings.pureEval
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path
).atPos(pos).debugThrow();
}
std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
{
auto & value = value0.s;
auto i = lookupPathResolved.find(value);
if (i != lookupPathResolved.end()) return i->second;
auto finish = [&](std::string res) {
debug("resolved search path element '%s' to '%s'", value, res);
lookupPathResolved.emplace(value, res);
return res;
};
if (EvalSettings::isPseudoUrl(value)) {
try {
auto accessor = fetchers::downloadTarball(
EvalSettings::resolvePseudoUrl(value)).accessor;
auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
return finish(store->toRealPath(storePath));
} catch (Error & e) {
logWarning({
2024-02-04 06:35:19 +02:00
.msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
});
}
}
if (auto colPos = value.find(':'); colPos != value.npos) {
auto scheme = value.substr(0, colPos);
auto rest = value.substr(colPos + 1);
if (auto * hook = get(settings.lookupPathHooks, scheme)) {
auto res = (*hook)(store, rest);
if (res)
return finish(std::move(*res));
}
}
{
auto path = absPath(value);
/* Allow access to paths in the search path. */
if (initAccessControl) {
allowPath(path);
if (store->isInStore(path)) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(path).first, closure);
for (auto & p : closure)
allowPath(p);
} catch (InvalidPath &) { }
}
}
if (pathExists(path))
return finish(std::move(path));
else {
logWarning({
2024-02-04 06:35:19 +02:00
.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)
});
}
}
debug("failed to resolve search path element '%s'", value);
return std::nullopt;
}
Expr * EvalState::parse(
char * text,
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
std::shared_ptr<StaticEnv> & staticEnv)
{
auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols);
result->bindVars(*this, staticEnv);
return result;
}
std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const
{
state.error<TypeError>(
"cannot coerce %1% to a string: %2%", showType(), *this
).atPos(pos).debugThrow();
}
bool ExternalValueBase::operator==(const ExternalValueBase & b) const
{
return false;
}
std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
return v.print(str);
}
}