Merge remote-tracking branch 'origin/master' into substitute-other-storedir

.dir-locals.el

@@ -1,6 +1,7 @@
 ((c++-mode . (
   (c-file-style . "k&r")
   (c-basic-offset . 4)
+  (c-block-comment-prefix . "  ")
   (indent-tabs-mode . nil)
   (tab-width . 4)
   (show-trailing-whitespace . t)

.github/workflows/test.yml

@@ -12,3 +12,13 @@ jobs:
     - uses: actions/checkout@v2
     - uses: cachix/install-nix-action@v10
     - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+  macos_perf_test:
+    runs-on: macos-latest
+    steps:
+    - name: Disable syspolicy assessments
+      run: |
+        spctl --status
+        sudo spctl --master-disable
+    - uses: actions/checkout@v2
+    - uses: cachix/install-nix-action@v10
+    - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings

maintainers/upload-release.pl

@@ -170,15 +170,5 @@ $channelsBucket->add_key(
 chdir("/home/eelco/Dev/nix-pristine") or die;
 system("git remote update origin") == 0 or die;
 system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
+system("git push --tags") == 0 or die;
-# Update the website.
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
-           "[%-\n" .
-           "latestNixVersion = \"$version\"\n" .
-           "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;

mk/tests.mk

@@ -1,11 +1,14 @@
 # Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
 define run-install-test
 
   installcheck: $1.test
 
   .PHONY: $1.test
-  $1.test: $1 tests/common.sh tests/init.sh
+  $1.test: $1 $(test-deps)
-	@env TEST_NAME=$1 TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
+	@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
 
 endef
 

src/libexpr/attr-path.cc

@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
 
     Symbol file = state.symbols.create(filename);
 
-    return { file, lineno, 0 };
+    return { foFile, file, lineno, 0 };
 }
 
 

src/libexpr/attr-set.hh

@@ -78,7 +78,7 @@ public:
         if (!a)
             throw Error({
                 .hint = hintfmt("attribute '%s' missing", name),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         return *a;

src/libexpr/eval-inline.hh

@@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
 {
     throw EvalError({
         .hint = hintfmt(s),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, showType(v)),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/eval.cc

@@ -529,7 +529,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -542,7 +542,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
 {
     throw EvalError({
         .hint = hintfmt(s, s2, s3),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -551,7 +551,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
     // p1 is where the error occurred; p2 is a position mentioned in the message.
     throw EvalError({
         .hint = hintfmt(s, sym, p2),
-        .nixCode = NixCode { .errPos = p1 }
+        .errPos = p1
     });
 }
 
@@ -559,7 +559,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
 {
     throw TypeError({
         .hint = hintfmt(s),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -572,7 +572,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
 {
     throw TypeError({
         .hint = hintfmt(s, fun.showNamePos(), s2),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -580,7 +580,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
 {
     throw AssertionError({
         .hint = hintfmt(s, s1),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
@@ -588,23 +588,18 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
 {
     throw UndefinedVarError({
         .hint = hintfmt(s, s1),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % s2);
+    e.addTrace(std::nullopt, s, s2);
 }
 
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
 {
-    e.addPrefix(format(s) % fun.showNamePos() % pos);
+    e.addTrace(pos, s, s2);
-}
-
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
-{
-    e.addPrefix(format(s) % s2 % pos);
 }
 
 
@@ -818,7 +813,7 @@ void EvalState::evalFile(const Path & path_, Value & v)
     try {
         eval(e, v);
     } catch (Error & e) {
-        addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
+        addErrorTrace(e, "while evaluating the file '%1%':", path2);
         throw;
     }
 
@@ -1068,8 +1063,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
 
     } catch (Error & e) {
         if (pos2 && pos2->file != state.sDerivationNix)
-            addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
+            addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
-                showAttrPath(state, env, attrPath), *pos2);
+                showAttrPath(state, env, attrPath));
         throw;
     }
 
@@ -1237,11 +1232,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
 
     /* Evaluate the body.  This is conditional on showTrace, because
        catching exceptions makes this function not tail-recursive. */
-    if (settings.showTrace)
+    if (loggerSettings.showTrace.get())
         try {
             lambda.body->eval(*this, env2, v);
         } catch (Error & e) {
-            addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
+            addErrorTrace(e, lambda.pos, "while evaluating %s", 
+              (lambda.name.set() 
+                  ? "'" + (string) lambda.name + "'" 
+                  : "anonymous lambdaction"));
+            addErrorTrace(e, pos, "from call site%s", "");
             throw;
         }
     else
@@ -1516,7 +1515,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     recurse(*i.value);
                 } catch (Error & e) {
-                    addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
+                    addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
                     throw;
                 }
         }
@@ -1936,7 +1935,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
 {
     throw TypeError({
         .hint = hintfmt("cannot coerce %1% to a string", showType()),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/eval.hh

@@ -250,7 +250,7 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(const char * text, const Path & path,
+    Expr * parse(const char * text, FileOrigin origin, const Path & path,
         const Path & basePath, StaticEnv & staticEnv);
 
 public:

src/libexpr/nixexpr.cc

@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
     if (!pos)
         str << "undefined position";
     else
-        str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+    {
+        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+        switch (pos.origin) {
+            case foFile:
+                f % (string) pos.file;
+                break;
+            case foStdin:
+            case foString:
+                f % "(string)";
+                break;
+            default:
+                throw Error("unhandled Pos origin!");
+        }
+        str << (f % pos.line % pos.column).str();
+    }
+
     return str;
 }
 
@@ -270,7 +285,7 @@ void ExprVar::bindVars(const StaticEnv & env)
     if (withLevel == -1)
         throw UndefinedVarError({
             .hint = hintfmt("undefined variable '%1%'", name),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     fromWith = true;
     this->level = withLevel;

src/libexpr/nixexpr.hh

@@ -24,11 +24,12 @@ MakeError(RestrictedPathError, Error);
 
 struct Pos
 {
+    FileOrigin origin;
     Symbol file;
     unsigned int line, column;
-    Pos() : line(0), column(0) { };
+    Pos() : origin(foString), line(0), column(0) { };
-    Pos(const Symbol & file, unsigned int line, unsigned int column)
+    Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
-        : file(file), line(line), column(column) { };
+        : origin(origin), file(file), line(line), column(column) { };
     operator bool() const
     {
         return line != 0;
@@ -238,7 +239,7 @@ struct ExprLambda : Expr
         if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
             throw ParseError({
                 .hint = hintfmt("duplicate formal function argument '%1%'", arg),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
     };
     void setName(Symbol & name);

src/libexpr/parser.y

@@ -30,7 +30,8 @@ namespace nix {
         SymbolTable & symbols;
         Expr * result;
         Path basePath;
-        Symbol path;
+        Symbol file;
+        FileOrigin origin;
         ErrorInfo error;
         Symbol sLetBody;
         ParseData(EvalState & state)
@@ -65,18 +66,17 @@ namespace nix {
 static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
-        .hint = hintfmt("attribute '%1%' already defined at %2%",
+         .hint = hintfmt("attribute '%1%' already defined at %2%",
             showAttrPath(attrPath), prevPos),
-        .nixCode = NixCode { .errPos = pos },
+         .errPos = pos
     });
 }
 
-
 static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
 {
     throw ParseError({
         .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
-        .nixCode = NixCode { .errPos = pos },
+        .errPos = pos
     });
 }
 
@@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
         throw ParseError({
             .hint = hintfmt("duplicate formal function argument '%1%'",
                 formal.name),
-            .nixCode = NixCode { .errPos = pos },
+            .errPos = pos
         });
     formals->formals.push_front(formal);
 }
@@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
 
 static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
 {
-    return Pos(data->path, loc.first_line, loc.first_column);
+    return Pos(data->origin, data->file, loc.first_line, loc.first_column);
 }
 
 #define CUR_POS makeCurPos(*yylocp, data)
@@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
 {
     data->error = {
         .hint = hintfmt(error),
-        .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
+        .errPos = makeCurPos(*loc, data)
     };
 }
 
@@ -339,7 +339,7 @@ expr_function
     { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .hint = hintfmt("dynamic attributes not allowed in let"),
-            .nixCode = NixCode { .errPos = CUR_POS },
+            .errPos = CUR_POS
         });
       $$ = new ExprLet($2, $4);
     }
@@ -419,7 +419,7 @@ expr_simple
       if (noURLLiterals)
           throw ParseError({
               .hint = hintfmt("URL literals are disabled"),
-              .nixCode = NixCode { .errPos = CUR_POS }
+              .errPos = CUR_POS
           });
       $$ = new ExprString(data->symbols.create($1));
   }
@@ -492,7 +492,7 @@ attrs
       } else
           throw ParseError({
               .hint = hintfmt("dynamic attributes not allowed in inherit"),
-              .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+              .errPos = makeCurPos(@2, data)
           });
     }
   | { $$ = new AttrPath; }
@@ -569,13 +569,24 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
     const Path & path, const Path & basePath, StaticEnv & staticEnv)
 {
     yyscan_t scanner;
     ParseData data(*this);
+    data.origin = origin;
+    switch (origin) {
+        case foFile: 
+            data.file = data.symbols.create(path);
+            break;
+        case foStdin:
+        case foString:
+            data.file = data.symbols.create(text);
+            break;
+        default:
+            assert(false);
+    }
     data.basePath = basePath;
-    data.path = data.symbols.create(path);
 
     yylex_init(&scanner);
     yy_scan_string(text, scanner);
@@ -625,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
 
 Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
 {
-    return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+    return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
 }
 
 
 Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
 {
-    return parse(s.data(), "(string)", basePath, staticEnv);
+    return parse(s.data(), foString, "", basePath, staticEnv);
 }
 
 
@@ -644,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
 Expr * EvalState::parseStdin()
 {
     //Activity act(*logger, lvlTalkative, format("parsing standard input"));
-    return parseExprFromString(drainFD(0), absPath("."));
+    return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
 }
 
 
@@ -693,7 +704,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
             ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
             : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
             path),
-        .nixCode = NixCode { .errPos = pos }
+        .errPos = pos
     });
 }
 

src/libexpr/primops.cc

@@ -96,7 +96,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -177,7 +177,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
             .hint = hintfmt(
                 "cannot import '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -215,7 +215,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (count == 0) {
         throw EvalError({
             .hint = hintfmt("at least one argument to 'exec' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
     PathSet context;
@@ -230,7 +230,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
         throw EvalError({
             .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
                 program, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -239,13 +239,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
     try {
         parsed = state.parseExprFromString(output, pos.file);
     } catch (Error & e) {
-        e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
+        e.addTrace(pos, "While parsing the output from '%1%'", program);
         throw;
     }
     try {
         state.eval(parsed, v);
     } catch (Error & e) {
-        e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
+        e.addTrace(pos, "While evaluating the output from '%1%'", program);
         throw;
     }
 }
@@ -385,7 +385,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (startSet == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'startSet' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceList(*startSet->value, pos);
 
@@ -399,7 +399,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
     if (op == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute 'operator' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceValue(*op->value, pos);
 
@@ -421,7 +421,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
         if (key == e->attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'key' required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         state.forceValue(*key->value, pos);
 
@@ -471,7 +471,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
+        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
         throw;
     }
 }
@@ -556,14 +556,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (attr == args[0]->attrs->end())
         throw EvalError({
             .hint = hintfmt("required attribute 'name' missing"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     string drvName;
     Pos & posDrvName(*attr->pos);
     try {
         drvName = state.forceStringNoCtx(*attr->value, pos);
     } catch (Error & e) {
-        e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
+        e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
         throw;
     }
 
@@ -603,7 +603,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             else
                 throw EvalError({
                     .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
-                    .nixCode = NixCode { .errPos = posDrvName }
+                    .errPos = posDrvName
                 });
         };
 
@@ -613,7 +613,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (outputs.find(j) != outputs.end())
                     throw EvalError({
                         .hint = hintfmt("duplicate derivation output '%1%'", j),
-                        .nixCode = NixCode { .errPos = posDrvName }
+                        .errPos = posDrvName
                     });
                 /* !!! Check whether j is a valid attribute
                    name. */
@@ -623,14 +623,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
                 if (j == "drv")
                     throw EvalError({
                         .hint = hintfmt("invalid derivation output name 'drv'" ),
-                        .nixCode = NixCode { .errPos = posDrvName }
+                        .errPos = posDrvName
                     });
                 outputs.insert(j);
             }
             if (outputs.empty())
                 throw EvalError({
                     .hint = hintfmt("derivation cannot have an empty set of outputs"),
-                    .nixCode = NixCode { .errPos = posDrvName }
+                    .errPos = posDrvName
                 });
         };
 
@@ -696,8 +696,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
             }
 
         } catch (Error & e) {
-            e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
+            e.addTrace(posDrvName, 
-                % key % drvName % posDrvName);
+                "while evaluating the attribute '%1%' of the derivation '%2%'",
+                key, drvName);
             throw;
         }
     }
@@ -745,20 +746,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
     if (drv.builder == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'builder' missing"),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     if (drv.platform == "")
         throw EvalError({
             .hint = hintfmt("required attribute 'system' missing"),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     /* Check whether the derivation name is valid. */
     if (isDerivation(drvName))
         throw EvalError({
             .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
-            .nixCode = NixCode { .errPos = posDrvName }
+            .errPos = posDrvName
         });
 
     if (outputHash) {
@@ -766,7 +767,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
         if (outputs.size() != 1 || *(outputs.begin()) != "out")
             throw Error({
                 .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
-                .nixCode = NixCode { .errPos = posDrvName }
+                .errPos = posDrvName
             });
 
         std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
@@ -880,7 +881,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
     if (!state.store->isInStore(path))
         throw EvalError({
             .hint = hintfmt("path '%1%' is not in the Nix store", path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     Path path2 = state.store->toStorePath(path);
     if (!settings.readOnlyMode)
@@ -901,7 +902,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
             .hint = hintfmt(
                 "cannot check the existence of '%1%', since path '%2%' is not valid",
                 path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -947,7 +948,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
     string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
@@ -978,7 +979,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         if (i == v2.attrs->end())
             throw EvalError({
                 .hint = hintfmt("attribute 'path' missing"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         PathSet context;
@@ -989,7 +990,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
         } catch (InvalidPathError & e) {
             throw EvalError({
                 .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
 
@@ -1009,7 +1010,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
     if (!ht)
       throw Error({
           .hint = hintfmt("unknown hash type '%1%'", type),
-          .nixCode = NixCode { .errPos = pos }
+          .errPos = pos
       });
 
     PathSet context; // discarded
@@ -1028,7 +1029,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
     } catch (InvalidPathError & e) {
         throw EvalError({
             .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 
@@ -1104,7 +1105,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
                     "in 'toFile': the file named '%1%' must not contain a reference "
                     "to a derivation but contains (%2%)",
                     name, path),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         refs.insert(state.store->parseStorePath(path));
     }
@@ -1175,7 +1176,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
     if (!context.empty())
         throw EvalError({
             .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.forceValue(*args[0], pos);
@@ -1184,7 +1185,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
             .hint = hintfmt(
                 "first argument in call to 'filterSource' is not a function but %1%",
                 showType(*args[0])),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
@@ -1207,7 +1208,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
             if (!context.empty())
                 throw EvalError({
                     .hint = hintfmt("string '%1%' cannot refer to other paths", path),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         } else if (attr.name == state.sName)
             name = state.forceStringNoCtx(*attr.value, *attr.pos);
@@ -1221,13 +1222,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
         else
             throw EvalError({
                 .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
-                .nixCode = NixCode { .errPos = *attr.pos }
+                .errPos = *attr.pos
             });
     }
     if (path.empty())
         throw EvalError({
             .hint = hintfmt("'path' required"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     if (name.empty())
         name = baseNameOf(path);
@@ -1288,7 +1289,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
     if (i == args[1]->attrs->end())
         throw EvalError({
             .hint = hintfmt("attribute '%1%' missing", attr),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     // !!! add to stack trace?
     if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
@@ -1371,7 +1372,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
         if (j == v2.attrs->end())
             throw TypeError({
                 .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         string name = state.forceStringNoCtx(*j->value, pos);
 
@@ -1381,7 +1382,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
             if (j2 == v2.attrs->end())
                 throw TypeError({
                     .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
-                    .nixCode = NixCode { .errPos = pos }
+                    .errPos = pos
                 });
             v.attrs->push_back(Attr(sym, j2->value, j2->pos));
         }
@@ -1457,7 +1458,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
     if (args[0]->type != tLambda)
         throw TypeError({
             .hint = hintfmt("'functionArgs' requires a function"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     if (!args[0]->lambda.fun->matchAttrs) {
@@ -1513,7 +1514,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
     if (n < 0 || (unsigned int) n >= list.listSize())
         throw Error({
             .hint = hintfmt("list index %1% is out of bounds", n),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     state.forceValue(*list.listElems()[n], pos);
     v = *list.listElems()[n];
@@ -1543,7 +1544,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
     if (args[0]->listSize() == 0)
         throw Error({
             .hint = hintfmt("'tail' called on an empty list"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.mkList(v, args[0]->listSize() - 1);
@@ -1688,7 +1689,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
     if (len < 0)
         throw EvalError({
             .hint = hintfmt("cannot create list of size %1%", len),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     state.mkList(v, len);
@@ -1850,7 +1851,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
     if (f2 == 0)
         throw EvalError({
             .hint = hintfmt("division by zero"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     if (args[0]->type == tFloat || args[1]->type == tFloat) {
@@ -1862,7 +1863,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
         if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
             throw EvalError({
                 .hint = hintfmt("overflow in integer division"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         mkInt(v, i1 / i2);
@@ -1923,7 +1924,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
     if (start < 0)
         throw EvalError({
             .hint = hintfmt("negative start position in 'substring'"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
@@ -1946,7 +1947,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
     if (!ht)
         throw Error({
             .hint = hintfmt("unknown hash type '%1%'", type),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     PathSet context; // discarded
@@ -1992,12 +1993,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
     }
@@ -2065,12 +2066,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
             // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
             throw EvalError({
                 .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         } else {
             throw EvalError({
                 .hint = hintfmt("invalid regular expression '%s'", re),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
         }
     }
@@ -2104,7 +2105,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
     if (args[0]->listSize() != args[1]->listSize())
         throw EvalError({
             .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
 
     vector<string> from;

src/libexpr/primops/context.cc

@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
         if (!state.store->isStorePath(i.name))
             throw EvalError({
                 .hint = hintfmt("Context key '%s' is not a store path", i.name),
-                .nixCode = NixCode { .errPos = *i.pos }
+                .errPos = *i.pos
             });
         if (!settings.readOnlyMode)
             state.store->ensurePath(state.store->parseStorePath(i.name));
@@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
                 if (!isDerivation(i.name)) {
                     throw EvalError({
                         .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
-                        .nixCode = NixCode { .errPos = *i.pos }
+                        .errPos = *i.pos
                     });
                 }
                 context.insert("=" + string(i.name));
@@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
             if (iter->value->listSize() && !isDerivation(i.name)) {
                 throw EvalError({
                     .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
-                    .nixCode = NixCode { .errPos = *i.pos }
+                    .errPos = *i.pos
                 });
             }
             for (unsigned int n = 0; n < iter->value->listSize(); ++n) {

src/libexpr/primops/fetchGit.cc

@@ -37,14 +37,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
     } else

src/libexpr/primops/fetchMercurial.cc

@@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
         }
 
         if (url.empty())
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
     } else

src/libexpr/primops/fetchTree.cc

@@ -68,7 +68,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
         if (!attrs.count("type"))
             throw Error({
                 .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
 
         input = fetchers::inputFromAttrs(attrs);
@@ -112,14 +112,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
             else
                 throw EvalError({
                     .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
-                    .nixCode = NixCode { .errPos = *attr.pos }
+                    .errPos = *attr.pos
                 });
             }
 
         if (!url)
             throw EvalError({
                 .hint = hintfmt("'url' argument required"),
-                .nixCode = NixCode { .errPos = pos }
+                .errPos = pos
             });
     } else
         url = state.forceStringNoCtx(*args[0], pos);

src/libexpr/primops/fromTOML.cc

@@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
     } catch (std::runtime_error & e) {
         throw EvalError({
             .hint = hintfmt("while parsing a TOML string: %s", e.what()),
-            .nixCode = NixCode { .errPos = pos }
+            .errPos = pos
         });
     }
 }

src/libfetchers/cache.hh

@@ -6,6 +6,8 @@ namespace nix::fetchers {
 
 struct Cache
 {
+    virtual ~Cache() { }
+
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libmain/loggers.cc

@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
     case LogFormat::rawWithLogs:
         return makeSimpleLogger(true);
     case LogFormat::internalJson:
-        return makeJSONLogger(*makeSimpleLogger());
+        return makeJSONLogger(*makeSimpleLogger(true));
     case LogFormat::bar:
         return makeProgressBar();
     case LogFormat::barWithLogs:

src/libmain/progress-bar.cc

@@ -131,7 +131,7 @@ public:
         auto state(state_.lock());
 
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         log(*state, ei.level, oss.str());
     }

src/libmain/shared.cc

@@ -323,10 +323,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
         printError("Try '%1% --help' for more information.", programName);
         return 1;
     } catch (BaseError & e) {
-        if (settings.showTrace && e.prefix() != "")
-            printError(e.prefix());
         logError(e.info());
-        if (e.prefix() != "" && !settings.showTrace)
+        if (e.hasTrace() && !loggerSettings.showTrace.get())
             printError("(use '--show-trace' to show detailed location information)");
         return e.status;
     } catch (std::bad_alloc & e) {

src/libstore/build.cc

@@ -2041,7 +2041,10 @@ void DerivationGoal::startBuilder()
             if (!std::regex_match(fileName, regex))
                 throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
 
-            auto storePath = worker.store.parseStorePath(*i++);
+            auto storePathS = *i++;
+            if (!worker.store.isInStore(storePathS))
+                throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
+            auto storePath = worker.store.parseStorePath(worker.store.toStorePath(storePathS));
 
             /* Write closure info to <fileName>. */
             writeFile(tmpDir + "/" + fileName,

src/libstore/content-address.cc

@@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
     return ca ? renderContentAddress(*ca) : "";
 }
 
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+    return std::visit(overloaded {
+        [](TextHash th) {
+            return th.hash;
+        },
+        [](FixedOutputHash fsh) {
+            return fsh.hash;
+        }
+    }, ca);
+}
+
 }

src/libstore/content-address.hh

@@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
 
 std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
 
+Hash getContentAddressHash(const ContentAddress & ca);
+
 }

src/libstore/daemon.cc

@@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
         if (ei.level > verbosity) return;
 
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, false);
 
         StringSink buf;
-        buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
+        buf << STDERR_NEXT << oss.str();
         enqueueMsg(*buf.s);
     }
 

src/libstore/globals.cc

@@ -35,7 +35,7 @@ Settings::Settings()
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
-    , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
+    , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
     buildUsersGroup = getuid() == 0 ? "nixbld" : "";
     lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";

src/libstore/globals.hh

@@ -196,10 +196,6 @@ public:
     /* Whether to lock the Nix client and worker to the same CPU. */
     bool lockCPU;
 
-    /* Whether to show a stack trace if Nix evaluation fails. */
-    Setting<bool> showTrace{this, false, "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
-
     Setting<SandboxMode> sandboxMode{this,
         #if __linux__
           smEnabled
@@ -366,6 +362,9 @@ public:
 
     Setting<bool> warnDirty{this, true, "warn-dirty",
         "Whether to warn about dirty Git/Mercurial trees."};
+
+    Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+        "Maximum size of NARs before spilling them to disk."};
 };
 
 

src/libstore/local-store.cc

@@ -988,7 +988,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
 
         PathLocks outputLock;
 
-        Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+        auto realPath = Store::toRealPath(info.path);
 
         /* Lock the output path.  But don't lock if we're being called
            from a build hook (whose parent process already acquired a
@@ -1059,8 +1059,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
         /* The first check above is an optimisation to prevent
            unnecessary lock acquisition. */
 
-        Path realPath = realStoreDir + "/";
+        auto realPath = Store::toRealPath(dstPath);
-        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 
@@ -1110,16 +1109,125 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
 {
     Path srcPath(absPath(_srcPath));
 
-    /* Read the whole path into memory. This is not a very scalable
+    /* For computing the NAR hash. */
-       method for very large paths, but `copyPath' is mainly used for
+    auto sha256Sink = std::make_unique<HashSink>(htSHA256);
-       small files. */
-    StringSink sink;
-    if (method == FileIngestionMethod::Recursive)
-        dumpPath(srcPath, sink, filter);
-    else
-        sink.s = make_ref<std::string>(readFile(srcPath));
 
-    return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
+    /* For computing the store path. In recursive SHA-256 mode, this
+       is the same as the NAR hash, so no need to do it again. */
+    std::unique_ptr<HashSink> hashSink =
+        method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+        ? nullptr
+        : std::make_unique<HashSink>(hashAlgo);
+
+    /* Read the source path into memory, but only if it's up to
+       narBufferSize bytes. If it's larger, write it to a temporary
+       location in the Nix store. If the subsequently computed
+       destination store path is already valid, we just delete the
+       temporary path. Otherwise, we move it to the destination store
+       path. */
+    bool inMemory = true;
+    std::string nar;
+
+    auto source = sinkToSource([&](Sink & sink) {
+
+        LambdaSink sink2([&](const unsigned char * buf, size_t len) {
+            (*sha256Sink)(buf, len);
+            if (hashSink) (*hashSink)(buf, len);
+
+            if (inMemory) {
+                if (nar.size() + len > settings.narBufferSize) {
+                    inMemory = false;
+                    sink << 1;
+                    sink((const unsigned char *) nar.data(), nar.size());
+                    nar.clear();
+                } else {
+                    nar.append((const char *) buf, len);
+                }
+            }
+
+            if (!inMemory) sink(buf, len);
+        });
+
+        if (method == FileIngestionMethod::Recursive)
+            dumpPath(srcPath, sink2, filter);
+        else
+            readFile(srcPath, sink2);
+    });
+
+    std::unique_ptr<AutoDelete> delTempDir;
+    Path tempPath;
+
+    try {
+        /* Wait for the source coroutine to give us some dummy
+           data. This is so that we don't create the temporary
+           directory if the NAR fits in memory. */
+        readInt(*source);
+
+        auto tempDir = createTempDir(realStoreDir, "add");
+        delTempDir = std::make_unique<AutoDelete>(tempDir);
+        tempPath = tempDir + "/x";
+
+        if (method == FileIngestionMethod::Recursive)
+            restorePath(tempPath, *source);
+        else
+            writeFile(tempPath, *source);
+
+    } catch (EndOfFile &) {
+        if (!inMemory) throw;
+        /* The NAR fits in memory, so we didn't do restorePath(). */
+    }
+
+    auto sha256 = sha256Sink->finish();
+
+    Hash hash = hashSink ? hashSink->finish().first : sha256.first;
+
+    auto dstPath = makeFixedOutputPath(method, hash, name);
+
+    addTempRoot(dstPath);
+
+    if (repair || !isValidPath(dstPath)) {
+
+        /* The first check above is an optimisation to prevent
+           unnecessary lock acquisition. */
+
+        auto realPath = Store::toRealPath(dstPath);
+
+        PathLocks outputLock({realPath});
+
+        if (repair || !isValidPath(dstPath)) {
+
+            deletePath(realPath);
+
+            autoGC();
+
+            if (inMemory) {
+                /* Restore from the NAR in memory. */
+                StringSource source(nar);
+                if (method == FileIngestionMethod::Recursive)
+                    restorePath(realPath, source);
+                else
+                    writeFile(realPath, source);
+            } else {
+                /* Move the temporary path we restored above. */
+                if (rename(tempPath.c_str(), realPath.c_str()))
+                    throw Error("renaming '%s' to '%s'", tempPath, realPath);
+            }
+
+            canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
+
+            optimisePath(realPath);
+
+            ValidPathInfo info(dstPath);
+            info.narHash = sha256.first;
+            info.narSize = sha256.second;
+            info.ca = FixedOutputHash { .method = method, .hash = hash };
+            registerValidPath(info);
+        }
+
+        outputLock.setDeletion(true);
+    }
+
+    return dstPath;
 }
 
 
@@ -1133,8 +1241,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
 
     if (repair || !isValidPath(dstPath)) {
 
-        Path realPath = realStoreDir + "/";
+        auto realPath = Store::toRealPath(dstPath);
-        realPath += dstPath.to_string();
 
         PathLocks outputLock({realPath});
 

src/libstore/remote-store.cc

@@ -8,6 +8,7 @@
 #include "derivations.hh"
 #include "pool.hh"
 #include "finally.hh"
+#include "logging.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -238,7 +239,7 @@ void RemoteStore::setOptions(Connection & conn)
         overrides.erase(settings.maxSilentTime.name);
         overrides.erase(settings.buildCores.name);
         overrides.erase(settings.useSubstitutes.name);
-        overrides.erase(settings.showTrace.name);
+        overrides.erase(loggerSettings.showTrace.name);
         conn.to << overrides.size();
         for (auto & i : overrides)
             conn.to << i.first << i.second.value;

src/libstore/store-api.cc

@@ -7,6 +7,7 @@
 #include "json.hh"
 #include "derivations.hh"
 #include "url.hh"
+#include "archive.hh"
 
 #include <future>
 
@@ -238,6 +239,40 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
 }
 
 
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+    FileIngestionMethod method, HashType hashAlgo,
+    std::optional<Hash> expectedCAHash)
+{
+    /* FIXME: inefficient: we're reading/hashing 'tmpFile' three
+       times. */
+
+    auto [narHash, narSize] = hashPath(htSHA256, srcPath);
+
+    auto hash = method == FileIngestionMethod::Recursive
+        ? hashAlgo == htSHA256
+          ? narHash
+          : hashPath(hashAlgo, srcPath).first
+        : hashFile(hashAlgo, srcPath);
+
+    if (expectedCAHash && expectedCAHash != hash)
+        throw Error("hash mismatch for '%s'", srcPath);
+
+    ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+    info.narHash = narHash;
+    info.narSize = narSize;
+    info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+    if (!isValidPath(info.path)) {
+        auto source = sinkToSource([&](Sink & sink) {
+            dumpPath(srcPath, sink);
+        });
+        addToStore(info, *source);
+    }
+
+    return info;
+}
+
+
 Store::Store(const Params & params)
     : Config(params)
     , state({(size_t) pathInfoCacheSize})

src/libstore/store-api.hh

@@ -455,6 +455,13 @@ public:
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
         PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
 
+    /* Copy the contents of a path to the store and register the
+       validity the resulting path, using a constant amount of
+       memory. */
+    ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
+        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+        std::optional<Hash> expectedCAHash = {});
+
     // FIXME: remove?
     virtual StorePath addToStoreFromDump(const string & dump, const string & name,
         FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)

src/libutil/error.cc

@@ -7,14 +7,11 @@
 
 namespace nix {
 
-
 const std::string nativeSystem = SYSTEM;
 
-// addPrefix is used for show-trace.  Strings added with addPrefix
+BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
-// will print ahead of the error itself.
-BaseError & BaseError::addPrefix(const FormatOrString & fs)
 {
-    prefix_ = fs.s + prefix_;
+    err.traces.push_front(Trace { .pos = e, .hint = hint});
     return *this;
 }
 
@@ -28,7 +25,7 @@ const string& BaseError::calcWhat() const
         err.name = sname();
 
         std::ostringstream oss;
-        oss << err;
+        showErrorInfo(oss, err, false);
         what_ = oss.str();
 
         return *what_;
@@ -56,28 +53,114 @@ string showErrPos(const ErrPos &errPos)
     }
 }
 
-// if nixCode contains lines of code, print them to the ostream, indicating the error column.
+std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
-void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
+{
+    if (errPos.line <= 0)
+        return std::nullopt;
+
+    if (errPos.origin == foFile) {
+        LinesOfCode loc;
+        try {
+            AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
+            if (!fd) {
+                logError(SysError("opening file '%1%'", errPos.file).info());
+                return std::nullopt;
+            }
+            else
+            {
+                // count the newlines.
+                int count = 0;
+                string line;
+                int pl = errPos.line - 1;
+                do
+                {
+                    line = readLine(fd.get());
+                    ++count;
+                    if (count < pl)
+                    {
+                        ;
+                    }
+                    else if (count == pl) {
+                        loc.prevLineOfCode = line;
+                    } else if (count == pl + 1) {
+                        loc.errLineOfCode = line;
+                    } else if (count == pl + 2) {
+                        loc.nextLineOfCode = line;
+                        break;
+                    }
+                } while (true);
+                return loc;
+            }
+        }
+        catch (EndOfFile &eof) {
+            if (loc.errLineOfCode.has_value())
+                return loc;
+            else
+                return std::nullopt;
+        }
+        catch (std::exception &e) {
+            printError("error reading nix file: %s\n%s", errPos.file, e.what());
+            return std::nullopt;
+        }
+    } else {
+        std::istringstream iss(errPos.file);
+        // count the newlines.
+        int count = 0;
+        string line;
+        int pl = errPos.line - 1;
+
+        LinesOfCode loc;
+
+        do
+        {
+            std::getline(iss, line);
+            ++count;
+            if (count < pl)
+            {
+                ;
+            }
+            else if (count == pl) {
+                loc.prevLineOfCode = line;
+            } else if (count == pl + 1) {
+                loc.errLineOfCode = line;
+            } else if (count == pl + 2) {
+                loc.nextLineOfCode = line;
+                break;
+            }
+
+            if (!iss.good())
+                break;
+        } while (true);
+
+        return loc;
+    }
+}
+
+// print lines of code to the ostream, indicating the error column.
+void printCodeLines(std::ostream &out,
+    const string &prefix,
+    const ErrPos &errPos,
+    const LinesOfCode &loc)
 {
     // previous line of code.
-    if (nixCode.prevLineOfCode.has_value()) {
+    if (loc.prevLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
+            prefix,
-                (nixCode.errPos.line - 1),
+            (errPos.line - 1),
-                *nixCode.prevLineOfCode);
+            *loc.prevLineOfCode);
     }
 
-    if (nixCode.errLineOfCode.has_value()) {
+    if (loc.errLineOfCode.has_value()) {
         // line of code containing the error.
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
+            prefix,
-                (nixCode.errPos.line),
+            (errPos.line),
-                *nixCode.errLineOfCode);
+            *loc.errLineOfCode);
         // error arrows for the column range.
-        if (nixCode.errPos.column > 0) {
+        if (errPos.column > 0) {
-            int start = nixCode.errPos.column;
+            int start = errPos.column;
             std::string spaces;
             for (int i = 0; i < start; ++i) {
                 spaces.append(" ");
@@ -87,23 +170,49 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
 
             out << std::endl
                 << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
-                    prefix,
+                prefix,
-                    spaces,
+                spaces,
-                    arrows);
+                arrows);
         }
     }
 
     // next line of code.
-    if (nixCode.nextLineOfCode.has_value()) {
+    if (loc.nextLineOfCode.has_value()) {
         out << std::endl
             << fmt("%1% %|2$5d|| %3%",
-                prefix,
+            prefix,
-                (nixCode.errPos.line + 1),
+            (errPos.line + 1),
-                *nixCode.nextLineOfCode);
+            *loc.nextLineOfCode);
     }
 }
 
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
+void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
+{
+    if (pos)
+    {
+        switch (pos.origin) {
+            case foFile: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
+                break;
+            }
+            case foString: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " from string" << ANSI_NORMAL;
+                break;
+            }
+            case foStdin: {
+                out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+                    ANSI_BLUE << " from stdin" << ANSI_NORMAL;
+                break;
+            }
+            default:
+                throw Error("invalid FileOrigin in errPos");
+        }
+    }
+}
+
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
 {
     auto errwidth = std::max<size_t>(getWindowSize().second, 20);
     string prefix = "";
@@ -158,8 +267,12 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         }
     }
 
-    auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    auto ndl = prefix.length()
-    auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+        + filterANSIEscapes(levelString, true).length()
+        + 7
+        + einfo.name.length()
+        + einfo.programName.value_or("").length();
+    auto dashwidth = std::max<int>(errwidth - ndl, 3);
 
     std::string dashes(dashwidth, '-');
 
@@ -179,16 +292,9 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
             einfo.programName.value_or(""));
 
     bool nl = false;  // intersperse newline between sections.
-    if (einfo.nixCode.has_value()) {
+    if (einfo.errPos.has_value() && (*einfo.errPos)) {
-        if (einfo.nixCode->errPos.file != "") {
+        out << prefix << std::endl;
-            // filename, line, column.
+        printAtPos(prefix, *einfo.errPos, out);
-            out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
-                prefix,
-                einfo.nixCode->errPos.file,
-                showErrPos(einfo.nixCode->errPos));
-        } else {
-            out << std::endl << fmt("%1%from command line argument", prefix);
-        }
         nl = true;
     }
 
@@ -200,12 +306,16 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         nl = true;
     }
 
-    // lines of code.
+    if (einfo.errPos.has_value() && (*einfo.errPos)) {
-    if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
+        auto loc = getCodeLines(*einfo.errPos);
-        if (nl)
+
-            out << std::endl << prefix;
+        // lines of code.
-        printCodeLines(out, prefix, *einfo.nixCode);
+        if (loc.has_value()) {
-        nl = true;
+            if (nl)
+                out << std::endl << prefix;
+            printCodeLines(out, prefix, *einfo.errPos, *loc);
+            nl = true;
+        }
     }
 
     // hint
@@ -216,6 +326,54 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
         nl = true;
     }
 
+    // traces
+    if (showTrace && !einfo.traces.empty())
+    {
+        const string tracetitle(" show-trace ");
+
+        int fill = errwidth - tracetitle.length();
+        int lw = 0;
+        int rw = 0;
+        const int min_dashes = 3;
+        if (fill > min_dashes * 2) {
+            if (fill % 2 != 0) {
+                lw = fill / 2;
+                rw = lw + 1;
+            }
+            else
+            {
+                lw = rw = fill / 2;
+            }
+        }
+        else
+            lw = rw = min_dashes;
+
+        if (nl)
+            out << std::endl << prefix;
+
+        out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
+
+        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
+        {
+            out << std::endl << prefix;
+            out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
+
+            if (iter->pos.has_value() && (*iter->pos)) {
+                auto pos = iter->pos.value();
+                out << std::endl << prefix;
+                printAtPos(prefix, pos, out);
+
+                auto loc = getCodeLines(pos);
+                if (loc.has_value())
+                {
+                    out << std::endl << prefix;
+                    printCodeLines(out, prefix, pos, *loc);
+                    out << std::endl << prefix;
+                }
+            }
+        }
+    }
+
     return out;
 }
 }

src/libutil/error.hh

@@ -1,8 +1,8 @@
 #pragma once
 
-
 #include "ref.hh"
 #include "types.hh"
+#include "fmt.hh"
 
 #include <cstring>
 #include <list>
@@ -10,7 +10,9 @@
 #include <map>
 #include <optional>
 
-#include "fmt.hh"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 /* Before 4.7, gcc's std::exception uses empty throw() specifiers for
  * its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -25,20 +27,20 @@ namespace nix {
 
 /*
 
-This file defines two main structs/classes used in nix error handling.
+   This file defines two main structs/classes used in nix error handling.
 
-ErrorInfo provides a standard payload of error information, with conversion to string
+   ErrorInfo provides a standard payload of error information, with conversion to string
-happening in the logger rather than at the call site.
+   happening in the logger rather than at the call site.
 
-BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+   BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
-an ErrorInfo.
+   an ErrorInfo.
 
-ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+   ErrorInfo structs are sent to the logger as part of an exception, or directly with the
-logError or logWarning macros.
+   logError or logWarning macros.
 
-See the error-demo.cc program for usage examples.
+   See the error-demo.cc program for usage examples.
 
-*/
+ */
 
 typedef enum {
     lvlError = 0,
@@ -50,11 +52,25 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
+typedef enum {
+    foFile,
+    foStdin,
+    foString
+} FileOrigin;
+
+// the lines of code surrounding an error.
+struct LinesOfCode {
+    std::optional<string> prevLineOfCode;
+    std::optional<string> errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
+
 // ErrPos indicates the location of an error in a nix file.
 struct ErrPos {
     int line = 0;
     int column = 0;
     string file;
+    FileOrigin origin;
 
     operator bool() const
     {
@@ -65,6 +81,7 @@ struct ErrPos {
     template <class P>
     ErrPos& operator=(const P &pos)
     {
+        origin = pos.origin;
         line = pos.line;
         column = pos.column;
         // is file symbol null?
@@ -82,11 +99,9 @@ struct ErrPos {
     }
 };
 
-struct NixCode {
+struct Trace {
-    ErrPos errPos;
+    std::optional<ErrPos> pos;
-    std::optional<string> prevLineOfCode;
+    hintformat hint;
-    std::optional<string> errLineOfCode;
-    std::optional<string> nextLineOfCode;
 };
 
 struct ErrorInfo {
@@ -94,19 +109,19 @@ struct ErrorInfo {
     string name;
     string description;
     std::optional<hintformat> hint;
-    std::optional<NixCode> nixCode;
+    std::optional<ErrPos> errPos;
+    std::list<Trace> traces;
 
     static std::optional<string> programName;
 };
 
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace);
 
 /* BaseError should generally not be caught, as it has Interrupted as
    a subclass. Catch Error instead. */
 class BaseError : public std::exception
 {
 protected:
-    string prefix_; // used for location traces etc.
     mutable ErrorInfo err;
 
     mutable std::optional<string> what_;
@@ -117,23 +132,23 @@ public:
 
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
-        : err { .level = lvlError,
+        : err {.level = lvlError,
-                .hint = hintfmt(args...)
+            .hint = hintfmt(args...)
-              }
+            }
         , status(status)
     { }
 
     template<typename... Args>
     BaseError(const std::string & fs, const Args & ... args)
-        : err { .level = lvlError,
+        : err {.level = lvlError,
-                .hint = hintfmt(fs, args...)
+            .hint = hintfmt(fs, args...)
-              }
+            }
     { }
 
     BaseError(hintformat hint)
-        : err { .level = lvlError,
+        : err {.level = lvlError,
-                .hint = hint
+            .hint = hint
-              }
+            }
     { }
 
     BaseError(ErrorInfo && e)
@@ -154,10 +169,17 @@ public:
 #endif
 
     const string & msg() const { return calcWhat(); }
-    const string & prefix() const { return prefix_; }
-    BaseError & addPrefix(const FormatOrString & fs);
-
     const ErrorInfo & info() { calcWhat(); return err; }
+
+    template<typename... Args>
+    BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
+    {
+        return addTrace(e, hintfmt(fs, args...));
+    }
+
+    BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
+
+    bool hasTrace() const { return !err.traces.empty(); }
 };
 
 #define MakeError(newClass, superClass) \
@@ -177,7 +199,7 @@ public:
 
     template<typename... Args>
     SysError(const Args & ... args)
-      :Error("")
+        : Error("")
     {
         errNo = errno;
         auto hf = hintfmt(args...);

src/libutil/hash.cc

@@ -370,7 +370,7 @@ string printHashType(HashType ht)
     default:
         // illegal hash type enum value internally, as opposed to external input
         // which should be validated with nice error message.
-        abort();
+        assert(false);
     }
 }
 

src/libutil/logging.cc

@@ -1,5 +1,6 @@
 #include "logging.hh"
 #include "util.hh"
+#include "config.hh"
 
 #include <atomic>
 #include <nlohmann/json.hpp>
@@ -7,6 +8,10 @@
 
 namespace nix {
 
+LoggerSettings loggerSettings;
+
+static GlobalConfig::Register r1(&loggerSettings);
+
 static thread_local ActivityId curActivity = 0;
 
 ActivityId getCurActivity()
@@ -72,7 +77,7 @@ public:
     void logEI(const ErrorInfo & ei) override
     {
         std::stringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         log(ei.level, oss.str());
     }
@@ -173,7 +178,7 @@ struct JSONLogger : Logger {
     void logEI(const ErrorInfo & ei) override
     {
         std::ostringstream oss;
-        oss << ei;
+        showErrorInfo(oss, ei, loggerSettings.showTrace.get());
 
         nlohmann::json json;
         json["action"] = "msg";

src/libutil/logging.hh

@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "error.hh"
+#include "config.hh"
 
 namespace nix {
 
@@ -34,6 +35,16 @@ typedef enum {
 
 typedef uint64_t ActivityId;
 
+struct LoggerSettings : Config
+{
+    Setting<bool> showTrace{this,
+        false,
+        "show-trace",
+        "Whether to show a stack trace on evaluation errors."};
+};
+
+extern LoggerSettings loggerSettings;
+
 class Logger
 {
     friend struct Activity;

src/libutil/tests/logging.cc

@@ -11,6 +11,13 @@ namespace nix {
      * logEI
      * --------------------------------------------------------------------------*/
 
+    const char *test_file =
+        "previous line of code\n"
+        "this is the problem line of code\n"
+        "next line of code\n";
+    const char *one_liner =
+        "this is the other problem line of code";
+
     TEST(logEI, catpuresBasicProperties) {
 
         MakeError(TestError, Error);
@@ -137,7 +144,6 @@ namespace nix {
      * logError
      * --------------------------------------------------------------------------*/
 
-
     TEST(logError, logErrorWithoutHintOrCode) {
         testing::internal::CaptureStderr();
 
@@ -152,7 +158,7 @@ namespace nix {
 
     TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create(test_file);
 
         testing::internal::CaptureStderr();
 
@@ -162,21 +168,16 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
+                .errPos = Pos(foString, problem_file, 02, 13),
-                    .errPos = Pos(problem_file, 40, 13),
+            });
-                    .prevLineOfCode = "previous line of code",
-                    .errLineOfCode = "this is the problem line of code",
-                    .nextLineOfCode = "next line of code",
-                }});
-
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n    39| previous line of code\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n    41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
-    TEST(logError, logErrorWithoutLinesOfCode) {
+    TEST(logError, logErrorWithInvalidFile) {
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create("invalid filename");
         testing::internal::CaptureStderr();
 
         logError({
@@ -185,28 +186,23 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
+                .errPos = Pos(foFile, problem_file, 02, 13)
-                    .errPos = Pos(problem_file, 40, 13)
+            });
-                }});
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
     TEST(logError, logErrorWithOnlyHintAndName) {
-        SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
         testing::internal::CaptureStderr();
 
         logError({
                 .name = "error name",
                 .hint = hintfmt("hint %1%", "only"),
-                .nixCode = NixCode {
+            });
-                    .errPos = Pos(problem_file, 40, 13)
-                }});
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n");
 
     }
 
@@ -219,18 +215,18 @@ namespace nix {
 
         logWarning({
                 .name = "name",
-                .description = "error description",
+                .description = "warning description",
                 .hint = hintfmt("there was a %1%", "warning"),
             });
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
     }
 
     TEST(logWarning, logWarningWithFileLineNumAndCode) {
 
         SymbolTable testTable;
-        auto problem_file = testTable.create("myfile.nix");
+        auto problem_file = testTable.create(test_file);
 
         testing::internal::CaptureStderr();
 
@@ -240,18 +236,73 @@ namespace nix {
                 .hint = hintfmt("this hint has %1% templated %2%!!",
                     "yellow",
                     "values"),
-                .nixCode = NixCode {
+                .errPos = Pos(foStdin, problem_file, 2, 13),
-                    .errPos = Pos(problem_file, 40, 13),
+            });
-                    .prevLineOfCode = std::nullopt,
-                    .errLineOfCode = "this is the problem line of code",
-                    .nextLineOfCode = std::nullopt
-                }});
 
 
         auto str = testing::internal::GetCapturedStderr();
-        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n    40| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+        ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
     }
 
+    /* ----------------------------------------------------------------------------
+     * traces
+     * --------------------------------------------------------------------------*/
+
+    TEST(addTrace, showTracesWithShowTrace) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create(test_file);
+        auto oneliner_file = testTable.create(one_liner);
+        auto invalidfilename = testTable.create("invalid filename");
+
+        auto e = AssertionError(ErrorInfo {
+                .name = "wat",
+                .description = "show-traces",
+                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .errPos = Pos(foString, problem_file, 2, 13),
+            });
+
+        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+        testing::internal::CaptureStderr();
+
+        loggerSettings.showTrace.assign(true);
+
+        logError(e.info());
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n     1| this is the other problem line of code\n      |                   \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+    }
+
+    TEST(addTrace, hideTracesWithoutShowTrace) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create(test_file);
+        auto oneliner_file = testTable.create(one_liner);
+        auto invalidfilename = testTable.create("invalid filename");
+
+        auto e = AssertionError(ErrorInfo {
+                .name = "wat",
+                .description = "hide traces",
+                .hint = hintfmt("it has been %1% days since our last error", "zero"),
+                .errPos = Pos(foString, problem_file, 2, 13),
+            });
+
+        e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+        e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+        e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+        testing::internal::CaptureStderr();
+
+        loggerSettings.showTrace.assign(false);
+
+        logError(e.info());
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n     1| previous line of code\n     2| this is the problem line of code\n      |             \x1B[31;1m^\x1B[0m\n     3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
+    }
+
+
     /* ----------------------------------------------------------------------------
      * hintfmt
      * --------------------------------------------------------------------------*/

src/nix-env/nix-env.cc

@@ -593,7 +593,7 @@ static void upgradeDerivations(Globals & globals,
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
+                e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName());
                 throw;
             }
         }
@@ -1185,7 +1185,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         } catch (AssertionError & e) {
             printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
+            e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
             throw;
         }
     }

src/nix-prefetch-url/nix-prefetch-url.cc

@@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
 
         /* If an expected hash is given, the file may already exist in
            the store. */
-        Hash hash, expectedHash(ht);
+        std::optional<Hash> expectedHash;
+        Hash hash;
         std::optional<StorePath> storePath;
         if (args.size() == 2) {
             expectedHash = Hash(args[1], ht);
             const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-            storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
+            storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
             if (store->isValidPath(*storePath))
-                hash = expectedHash;
+                hash = *expectedHash;
             else
                 storePath.reset();
         }
@@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
                     tmpFile = unpacked;
             }
 
-            /* FIXME: inefficient; addToStore() will also hash
+            const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-               this. */
-            hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
 
-            if (expectedHash != Hash(ht) && expectedHash != hash)
+            auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
-                throw Error("hash mismatch for '%1%'", uri);
+            storePath = info.path;
-
+            assert(info.ca);
-            const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+            hash = getContentAddressHash(*info.ca);
-
-            /* Copy the file to the Nix store. FIXME: if RemoteStore
-               implemented addToStoreFromDump() and downloadFile()
-               supported a sink, we could stream the download directly
-               into the Nix store. */
-            storePath = store->addToStore(name, tmpFile, recursive, ht);
-
-            assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
         }
 
         stopProgressBar();

src/nix-store/nix-store.cc

@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
    store. */
 static void opAddFixed(Strings opFlags, Strings opArgs)
 {
-    auto recursive = FileIngestionMethod::Flat;
+    auto method = FileIngestionMethod::Flat;
 
     for (auto & i : opFlags)
-        if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
+        if (i == "--recursive") method = FileIngestionMethod::Recursive;
         else throw UsageError("unknown flag '%1%'", i);
 
     if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
     opArgs.pop_front();
 
     for (auto & i : opArgs)
-        cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
+        std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
 }
 
 

src/nix/develop.cc

@@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path)
         R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
 
     static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
+        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
 
     static std::regex varRegex(
         "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
@@ -135,13 +135,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = hashDerivationModulo(*store, drv, true);
     auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput {
+    drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath });
-        .path = shellOutPath,
-        .hash = FixedOutputHash {
-            .method = FileIngestionMethod::Flat,
-            .hash = Hash { },
-        },
-    });
     drv.env["out"] = store->printStorePath(shellOutPath);
     auto shellDrvPath2 = writeDerivation(store, drv, drvName);
 

src/nix/repl.cc

@@ -211,12 +211,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
                 // input without clearing the input so far.
                 continue;
             } else {
-              printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+              printMsg(lvlError, e.msg());
             }
         } catch (Error & e) {
-            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+          printMsg(lvlError, e.msg());
         } catch (Interrupted & e) {
-            printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+          printMsg(lvlError, e.msg());
         }
 
         // We handled the current input fully, so we should clear it

src/nix/search.cc

@@ -216,7 +216,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
             } catch (AssertionError & e) {
             } catch (Error & e) {
                 if (!toplevel) {
-                    e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
+                    e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath);
                     throw;
                 }
             }

tests/common.sh.in

@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
+export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
 unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
@@ -76,7 +77,7 @@ startDaemon() {
     rm -f $NIX_STATE_DIR/daemon-socket/socket
     nix-daemon &
     for ((i = 0; i < 30; i++)); do
-        if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi
+        if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
         sleep 1
     done
     pidDaemon=$!

tests/local.mk

@@ -40,4 +40,4 @@ tests-environment = NIX_REMOTE= $(bash) -e
 
 clean-files += $(d)/common.sh
 
-installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT)

tests/misc.sh

@@ -16,6 +16,11 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
-echo $eval_res | grep "(string) (1:15)"
+echo $eval_arg_res | grep "at: (1:15) from string"
-echo $eval_res | grep "infinite recursion encountered"
+echo $eval_arg_res | grep "infinite recursion encountered"
+
+eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
+echo $eval_stdin_res | grep "at: (1:15) from stdin"
+echo $eval_stdin_res | grep "infinite recursion encountered"
+

tests/nix-shell.sh

@@ -55,3 +55,10 @@ chmod a+rx $TEST_ROOT/shell.shebang.rb
 
 output=$($TEST_ROOT/shell.shebang.rb abc ruby)
 [ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
+
+# Test 'nix develop'.
+nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+
+# Test 'nix print-dev-env'.
+source <(nix print-dev-env -f shell.nix shellDrv)
+[[ -n $stdenv ]]