Merge remote-tracking branch 'nixos/master'

boehmgc-coroutine-sp-fallback.diff

@@ -1,75 +0,0 @@
-diff --git a/darwin_stop_world.c b/darwin_stop_world.c
-index 3dbaa3fb..36a1d1f7 100644
---- a/darwin_stop_world.c
-+++ b/darwin_stop_world.c
-@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void)
-   int nthreads = 0;
-   word total_size = 0;
-   mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ;
-+  size_t stack_limit;
-   if (!EXPECT(GC_thr_initialized, TRUE))
-     GC_thr_init();
- 
-@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void)
-             GC_push_all_stack_sections(lo, hi, p->traced_stack_sect);
-           }
-           if (altstack_lo) {
-+            // When a thread goes into a coroutine, we lose its original sp until
-+            // control flow returns to the thread.
-+            // While in the coroutine, the sp points outside the thread stack,
-+            // so we can detect this and push the entire thread stack instead,
-+            // as an approximation.
-+            // We assume that the coroutine has similarly added its entire stack.
-+            // This could be made accurate by cooperating with the application
-+            // via new functions and/or callbacks.
-+            stack_limit = pthread_get_stacksize_np(p->id);
-+            if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack
-+              altstack_lo = altstack_hi - stack_limit;
-+            }
-+
-             total_size += altstack_hi - altstack_lo;
-             GC_push_all_stack(altstack_lo, altstack_hi);
-           }
-diff --git a/pthread_stop_world.c b/pthread_stop_world.c
-index 4b2c429..1fb4c52 100644
---- a/pthread_stop_world.c
-+++ b/pthread_stop_world.c
-@@ -781,4 +781,6 @@ GC_INNER void GC_push_all_stacks(void)
-     pthread_t self = pthread_self();
-     word total_size = 0;
-+    size_t stack_limit;
-+    pthread_attr_t pattr;
- 
-     if (!EXPECT(GC_thr_initialized, TRUE))
-@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void)
-           hi = p->altstack + p->altstack_size;
-           /* FIXME: Need to scan the normal stack too, but how ? */
-           /* FIXME: Assume stack grows down */
-+        } else {
-+          if (pthread_getattr_np(p->id, &pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_getattr_np failed!");
-+          }
-+          if (pthread_attr_getstacksize(&pattr, &stack_limit)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!");
-+          }
-+          if (pthread_attr_destroy(&pattr)) {
-+            ABORT("GC_push_all_stacks: pthread_attr_destroy failed!");
-+          }
-+          // When a thread goes into a coroutine, we lose its original sp until
-+          // control flow returns to the thread.
-+          // While in the coroutine, the sp points outside the thread stack,
-+          // so we can detect this and push the entire thread stack instead,
-+          // as an approximation.
-+          // We assume that the coroutine has similarly added its entire stack.
-+          // This could be made accurate by cooperating with the application
-+          // via new functions and/or callbacks.
-+          #ifndef STACK_GROWS_UP
-+            if (lo >= hi || lo < hi - stack_limit) { // sp outside stack
-+              lo = hi - stack_limit;
-+            }
-+          #else
-+          #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix."
-+          #endif
-         }
-         GC_push_all_stack_sections(lo, hi, traced_stack_sect);
- #       ifdef STACK_GROWS_UP

doc/manual/book.toml

@@ -1,6 +1,11 @@
+[book]
+title = "Nix Reference Manual"
+
 [output.html]
 additional-css = ["custom.css"]
 additional-js = ["redirects.js"]
+edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
+git-repository-url = "https://github.com/NixOS/nix"
 
 [preprocessor.anchors]
 renderers = ["html"]

doc/manual/src/glossary.md

@@ -3,18 +3,18 @@
   - [derivation]{#gloss-derivation}\
     A description of a build task. The result of a derivation is a
     store object. Derivations are typically specified in Nix expressions
-    using the [`derivation` primitive](language/derivations.md). These are
+    using the [`derivation` primitive](./language/derivations.md). These are
     translated into low-level *store derivations* (implicitly by
     `nix-env` and `nix-build`, or explicitly by `nix-instantiate`).
 
   - [content-addressed derivation]{#gloss-content-addressed-derivation}\
     A derivation which has the
-    [`__contentAddressed`](language/advanced-attributes.md#adv-attr-__contentAddressed)
+    [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed)
     attribute set to `true`.
 
   - [fixed-output derivation]{#gloss-fixed-output-derivation}\
     A derivation which includes the
-    [`outputHash`](language/advanced-attributes.md#adv-attr-outputHash) attribute.
+    [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute.
 
   - [store]{#gloss-store}\
     The location in the file system where store objects live. Typically
@@ -79,7 +79,7 @@
   - [substituter]{#gloss-substituter}\
     A *substituter* is an additional store from which Nix will
     copy store objects it doesn't have.  For details, see the
-    [`substituters` option](command-ref/conf-file.html#conf-substituters).
+    [`substituters` option](./command-ref/conf-file.md#conf-substituters).
 
   - [purity]{#gloss-purity}\
     The assumption that equal Nix derivations when run always produce
@@ -139,7 +139,7 @@
     An automatically generated store object that consists of a set of
     symlinks to “active” applications, i.e., other store paths. These
     are generated automatically by
-    [`nix-env`](command-ref/nix-env.md). See *profiles*.
+    [`nix-env`](./command-ref/nix-env.md). See *profiles*.
 
   - [profile]{#gloss-profile}\
     A symlink to the current *user environment* of a user, e.g.,
@@ -150,7 +150,9 @@
     store. It can contain regular files, directories and symbolic
     links.  NARs are generated and unpacked using `nix-store --dump`
     and `nix-store --restore`.
+
   - [`∅`]{#gloss-emtpy-set}\
     The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile.
+
   - [`ε`]{#gloss-epsilon}\
     The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute.

flake.lock

@@ -18,16 +18,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1657693803,
+        "lastModified": 1670461440,
-        "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=",
+        "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2",
+        "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-22.05-small",
+        "ref": "nixos-22.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,7 +1,7 @@
 {
   description = "The purely functional package manager - but super!";
 
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05-small";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small";
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
 
@@ -127,13 +127,9 @@
           });
 
         propagatedDeps =
-          [ ((boehmgc.override {
+          [ (boehmgc.override {
               enableLargeConfig = true;
-            }).overrideAttrs(o: {
+            })
-              patches = (o.patches or []) ++ [
-                ./boehmgc-coroutine-sp-fallback.diff
-              ];
-            }))
             nlohmann_json
           ];
       };

maintainers/upload-release.pl

@@ -115,10 +115,6 @@ sub downloadFile {
 
     write_file("$tmpFile.sha256", $sha256_actual);
 
-    if (! -e "$tmpFile.asc") {
-        system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n";
-    }
-
     return $sha256_expected;
 }
 
@@ -194,7 +190,7 @@ for my $fn (glob "$tmpDir/*") {
         my $configuration = ();
         $configuration->{content_type} = "application/octet-stream";
 
-        if ($fn =~ /.sha256|.asc|install/) {
+        if ($fn =~ /.sha256|install/) {
             # Text files
             $configuration->{content_type} = "text/plain";
         }

src/libcmd/repl.cc

@@ -215,17 +215,15 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
     out << dt.hint.str() << "\n";
 
     // prefer direct pos, but if noPos then try the expr.
-    auto pos = *dt.pos
+    auto pos = dt.pos
-        ? *dt.pos
+        ? dt.pos
-        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
+        : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]);
 
     if (pos) {
-        printAtPos(pos, out);
+        out << pos;
-
+        if (auto loc = pos->getCodeLines()) {
-        auto loc = getCodeLines(pos);
-        if (loc.has_value()) {
             out << "\n";
-            printCodeLines(out, "", pos, *loc);
+            printCodeLines(out, "", *pos, *loc);
             out << "\n";
         }
     }
@@ -589,15 +587,17 @@ bool NixRepl::processLine(std::string line)
         Value v;
         evalString(arg, v);
 
-        const auto [file, line] = [&] () -> std::pair<std::string, uint32_t> {
+        const auto [path, line] = [&] () -> std::pair<Path, uint32_t> {
             if (v.type() == nPath || v.type() == nString) {
                 PathSet context;
-                auto filename = state->coerceToString(noPos, v, context).toOwned();
+                auto path = state->coerceToPath(noPos, v, context);
-                state->symbols.create(filename);
+                return {path, 0};
-                return {filename, 0};
             } else if (v.isLambda()) {
                 auto pos = state->positions[v.lambda.fun->pos];
-                return {pos.file, pos.line};
+                if (auto path = std::get_if<Path>(&pos.origin))
+                    return {*path, pos.line};
+                else
+                    throw Error("'%s' cannot be shown in an editor", pos);
             } else {
                 // assume it's a derivation
                 return findPackageFilename(*state, v, arg);
@@ -605,7 +605,7 @@ bool NixRepl::processLine(std::string line)
         }();
 
         // Open in EDITOR
-        auto args = editorFor(file, line);
+        auto args = editorFor(path, line);
         auto editor = args.front();
         args.pop_front();
 

src/libexpr/eval.cc

@@ -820,7 +820,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
         ? std::make_unique<DebugTraceStacker>(
             *this,
             DebugTrace {
-                .pos = error->info().errPos ? *error->info().errPos : positions[expr.getPos()],
+                .pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]),
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
@@ -1009,7 +1009,7 @@ void EvalState::throwMissingArgumentError(const PosIdx pos, const char * s, cons
 
 void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const
 {
-    e.addTrace(std::nullopt, s, s2);
+    e.addTrace(nullptr, s, s2);
 }
 
 void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2) const
@@ -1021,13 +1021,13 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::optional<ErrPos> pos,
+    std::shared_ptr<AbstractPos> && pos,
     const char * s,
     const std::string & s2)
 {
     return std::make_unique<DebugTraceStacker>(state,
         DebugTrace {
-            .pos = pos,
+            .pos = std::move(pos),
             .expr = expr,
             .env = env,
             .hint = hintfmt(s, s2),
@@ -1133,9 +1133,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr)
 void EvalState::mkPos(Value & v, PosIdx p)
 {
     auto pos = positions[p];
-    if (!pos.file.empty()) {
+    if (auto path = std::get_if<Path>(&pos.origin)) {
         auto attrs = buildBindings(3);
-        attrs.alloc(sFile).mkString(pos.file);
+        attrs.alloc(sFile).mkString(*path);
         attrs.alloc(sLine).mkInt(pos.line);
         attrs.alloc(sColumn).mkInt(pos.column);
         v.mkAttrs(attrs);
@@ -1243,7 +1243,7 @@ void EvalState::cacheFile(
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? std::optional(ErrPos(positions[e->getPos()])) : std::nullopt,
+                e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr,
                 "while evaluating the file '%1%':", resolvedPath)
             : nullptr;
 
@@ -1514,10 +1514,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
         state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) );
 
     } catch (Error & e) {
+        if (pos2) {
             auto pos2r = state.positions[pos2];
-        if (pos2 && pos2r.file != state.derivationNixPath)
+            auto origin = std::get_if<Path>(&pos2r.origin);
+            if (!(origin && *origin == state.derivationNixPath))
                 state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'",
                     showAttrPath(state, env, attrPath));
+        }
         throw;
     }
 
@@ -2494,7 +2497,8 @@ void EvalState::printStats()
                     else
                         obj["name"] = nullptr;
                     if (auto pos = positions[fun->pos]) {
-                        obj["file"] = (std::string_view) pos.file;
+                        if (auto path = std::get_if<Path>(&pos.origin))
+                            obj["file"] = *path;
                         obj["line"] = pos.line;
                         obj["column"] = pos.column;
                     }
@@ -2508,7 +2512,8 @@ void EvalState::printStats()
                 for (auto & i : attrSelects) {
                     json obj = json::object();
                     if (auto pos = positions[i.first]) {
-                        obj["file"] = (const std::string &) pos.file;
+                        if (auto path = std::get_if<Path>(&pos.origin))
+                            obj["file"] = *path;
                         obj["line"] = pos.line;
                         obj["column"] = pos.column;
                     }

src/libexpr/eval.hh

@@ -78,7 +78,7 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::optional<ErrPos> pos;
+    std::shared_ptr<AbstractPos> pos;
     const Expr & expr;
     const Env & env;
     hintformat hint;
@@ -457,8 +457,12 @@ private:
     friend struct ExprAttrs;
     friend struct ExprLet;
 
-    Expr * parse(char * text, size_t length, FileOrigin origin, const PathView path,
+    Expr * parse(
-        const PathView basePath, std::shared_ptr<StaticEnv> & staticEnv);
+        char * text,
+        size_t length,
+        Pos::Origin origin,
+        Path basePath,
+        std::shared_ptr<StaticEnv> & staticEnv);
 
 public:
 

src/libexpr/flake/config.cc

@@ -56,7 +56,7 @@ void ConfigFile::apply()
             auto tlname = get(trustedList, name);
             if (auto saved = tlname ? get(*tlname, valueS) : nullptr) {
                 trusted = *saved;
-                debug("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS);
+                printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS);
             } else {
                 // FIXME: filter ANSI escapes, newlines, \r, etc.
                 if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') {

src/libexpr/flake/flake.cc

@@ -220,7 +220,7 @@ static Flake getFlake(
     Value vInfo;
     state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
 
-    expectType(state, nAttrs, vInfo, state.positions.add({flakeFile, foFile}, 0, 0));
+    expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1));
 
     if (auto description = vInfo.attrs->get(state.sDescription)) {
         expectType(state, nString, *description->value, description->pos);

src/libexpr/nixexpr.cc

@@ -8,6 +8,58 @@
 
 namespace nix {
 
+struct PosAdapter : AbstractPos
+{
+    Pos::Origin origin;
+
+    PosAdapter(Pos::Origin origin)
+        : origin(std::move(origin))
+    {
+    }
+
+    std::optional<std::string> getSource() const override
+    {
+        return std::visit(overloaded {
+            [](const Pos::none_tag &) -> std::optional<std::string> {
+                return std::nullopt;
+            },
+            [](const Pos::Stdin & s) -> std::optional<std::string> {
+                // Get rid of the null terminators added by the parser.
+                return std::string(s.source->c_str());
+            },
+            [](const Pos::String & s) -> std::optional<std::string> {
+                // Get rid of the null terminators added by the parser.
+                return std::string(s.source->c_str());
+            },
+            [](const Path & path) -> std::optional<std::string> {
+                try {
+                    return readFile(path);
+                } catch (Error &) {
+                    return std::nullopt;
+                }
+            }
+        }, origin);
+    }
+
+    void print(std::ostream & out) const override
+    {
+        std::visit(overloaded {
+            [&](const Pos::none_tag &) { out << "«none»"; },
+            [&](const Pos::Stdin &) { out << "«stdin»"; },
+            [&](const Pos::String & s) { out << "«string»"; },
+            [&](const Path & path) { out << path; }
+        }, origin);
+    }
+};
+
+Pos::operator std::shared_ptr<AbstractPos>() const
+{
+    auto pos = std::make_shared<PosAdapter>(origin);
+    pos->line = line;
+    pos->column = column;
+    return pos;
+}
+
 /* Displaying abstract syntax trees. */
 
 static void showString(std::ostream & str, std::string_view s)
@@ -248,24 +300,10 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const
 
 std::ostream & operator << (std::ostream & str, const Pos & pos)
 {
-    if (!pos)
+    if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) {
+        str << *pos2;
+    } else
         str << "undefined position";
-    else
-    {
-        auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
-        switch (pos.origin) {
-            case foFile:
-                f % (const std::string &) pos.file;
-                break;
-            case foStdin:
-            case foString:
-                f % "(string)";
-                break;
-            default:
-                throw Error("unhandled Pos origin!");
-        }
-        str << (f % pos.line % pos.column).str();
-    }
 
     return str;
 }

src/libexpr/nixexpr.hh

@@ -23,15 +23,22 @@ MakeError(MissingArgumentError, EvalError);
 MakeError(RestrictedPathError, Error);
 
 /* Position objects. */
-
 struct Pos
 {
-    std::string file;
-    FileOrigin origin;
     uint32_t line;
     uint32_t column;
 
+    struct none_tag { };
+    struct Stdin { ref<std::string> source; };
+    struct String { ref<std::string> source; };
+
+    typedef std::variant<none_tag, Stdin, String, Path> Origin;
+
+    Origin origin;
+
     explicit operator bool() const { return line > 0; }
+
+    operator std::shared_ptr<AbstractPos>() const;
 };
 
 class PosIdx {
@@ -48,6 +55,10 @@ public:
     explicit operator bool() const { return id > 0; }
 
     bool operator <(const PosIdx other) const { return id < other.id; }
+
+    bool operator ==(const PosIdx other) const { return id == other.id; }
+
+    bool operator !=(const PosIdx other) const { return id != other.id; }
 };
 
 class PosTable
@@ -61,13 +72,13 @@ public:
         // current origins.back() can be reused or not.
         mutable uint32_t idx = std::numeric_limits<uint32_t>::max();
 
-        explicit Origin(uint32_t idx): idx(idx), file{}, origin{} {}
+        // Used for searching in PosTable::[].
+        explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {}
 
     public:
-        const std::string file;
+        const Pos::Origin origin;
-        const FileOrigin origin;
 
-        Origin(std::string file, FileOrigin origin): file(std::move(file)), origin(origin) {}
+        Origin(Pos::Origin origin): origin(origin) {}
     };
 
     struct Offset {
@@ -107,7 +118,7 @@ public:
             [] (const auto & a, const auto & b) { return a.idx < b.idx; });
         const auto origin = *std::prev(pastOrigin);
         const auto offset = offsets[idx];
-        return {origin.file, origin.origin, offset.line, offset.column};
+        return {offset.line, offset.column, origin.origin};
     }
 };
 

src/libexpr/parser.y

@@ -34,11 +34,6 @@ namespace nix {
         Path basePath;
         PosTable::Origin origin;
         std::optional<ErrorInfo> error;
-        ParseData(EvalState & state, PosTable::Origin origin)
-            : state(state)
-            , symbols(state.symbols)
-            , origin(std::move(origin))
-            { };
     };
 
     struct ParserFormals {
@@ -651,24 +646,20 @@ formal
 namespace nix {
 
 
-Expr * EvalState::parse(char * text, size_t length, FileOrigin origin,
+Expr * EvalState::parse(
-    const PathView path, const PathView basePath, std::shared_ptr<StaticEnv> & staticEnv)
+    char * text,
+    size_t length,
+    Pos::Origin origin,
+    Path basePath,
+    std::shared_ptr<StaticEnv> & staticEnv)
 {
     yyscan_t scanner;
-    std::string file;
+    ParseData data {
-    switch (origin) {
+        .state = *this,
-        case foFile:
+        .symbols = symbols,
-            file = path;
+        .basePath = std::move(basePath),
-            break;
+        .origin = {origin},
-        case foStdin:
+    };
-        case foString:
-            file = text;
-            break;
-        default:
-            assert(false);
-    }
-    ParseData data(*this, {file, origin});
-    data.basePath = basePath;
 
     yylex_init(&scanner);
     yy_scan_buffer(text, length, scanner);
@@ -720,14 +711,15 @@ Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv
     auto buffer = readFile(path);
     // readFile should have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv);
+    return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv);
 }
 
 
-Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv)
+Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv)
 {
-    s.append("\0\0", 2);
+    auto s = make_ref<std::string>(std::move(s_));
-    return parse(s.data(), s.size(), foString, "", basePath, staticEnv);
+    s->append("\0\0", 2);
+    return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv);
 }
 
 
@@ -743,7 +735,8 @@ Expr * EvalState::parseStdin()
     auto buffer = drainFD(0);
     // drainFD should have left some extra space for terminators
     buffer.append("\0\0", 2);
-    return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv);
+    auto s = make_ref<std::string>(std::move(buffer));
+    return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv);
 }
 
 

src/libexpr/primops.cc

@@ -368,8 +368,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v)
     auto output = runProgram(program, true, commandArgs);
     Expr * parsed;
     try {
-        auto base = state.positions[pos];
+        parsed = state.parseExprFromString(std::move(output), "/");
-        parsed = state.parseExprFromString(std::move(output), base.file);
     } catch (Error & e) {
         e.addTrace(state.positions[pos], "While parsing the output from '%1%'", program);
         throw;
@@ -798,7 +797,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         v = *args[1];
     } catch (Error & e) {
         PathSet context;
-        e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context).toOwned());
+        e.addTrace(nullptr, state.coerceToString(pos, *args[0], context).toOwned());
         throw;
     }
 }
@@ -4021,7 +4020,7 @@ void EvalState::createBaseEnv()
         // the parser needs two NUL bytes as terminators; one of them
         // is implied by being a C string.
         "\0";
-    eval(parse(code, sizeof(code), foFile, derivationNixPath, "/", staticBaseEnv), *vDerivation);
+    eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation);
 }
 
 

src/libexpr/tests/libexprtests.hh

@@ -123,7 +123,7 @@ namespace nix {
 
     MATCHER_P(IsAttrsOfSize, n, fmt("Is a set of size [%1%]", n)) {
         if (arg.type() != nAttrs) {
-            *result_listener << "Expexted set got " << arg.type();
+            *result_listener << "Expected set got " << arg.type();
             return false;
         } else if (arg.attrs->size() != (size_t)n) {
             *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size();

src/libexpr/tests/primops.cc

@@ -151,20 +151,7 @@ namespace nix {
         // The `y` attribute is at position
         const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }";
         auto v = eval(expr);
-        ASSERT_THAT(v, IsAttrsOfSize(3));
+        ASSERT_THAT(v, IsNull());
-
-        auto file = v.attrs->find(createSymbol("file"));
-        ASSERT_NE(file, nullptr);
-        // FIXME: The file when running these tests is the input string?!?
-        ASSERT_THAT(*file->value, IsStringEq(expr));
-
-        auto line = v.attrs->find(createSymbol("line"));
-        ASSERT_NE(line, nullptr);
-        ASSERT_THAT(*line->value, IsIntEq(1));
-
-        auto column = v.attrs->find(createSymbol("column"));
-        ASSERT_NE(column, nullptr);
-        ASSERT_THAT(*column->value, IsIntEq(33));
     }
 
     TEST_F(PrimOpTest, hasAttr) {
@@ -617,7 +604,7 @@ namespace nix {
 
     TEST_F(PrimOpTest, storeDir) {
         auto v = eval("builtins.storeDir");
-        ASSERT_THAT(v, IsStringEq("/nix/store"));
+        ASSERT_THAT(v, IsStringEq(settings.nixStore));
     }
 
     TEST_F(PrimOpTest, nixVersion) {

src/libexpr/value-to-xml.cc

@@ -24,7 +24,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
 
 static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos)
 {
-    xmlAttrs["path"] = pos.file;
+    if (auto path = std::get_if<Path>(&pos.origin))
+        xmlAttrs["path"] = *path;
     xmlAttrs["line"] = (format("%1%") % pos.line).str();
     xmlAttrs["column"] = (format("%1%") % pos.column).str();
 }

src/libstore/nar-info-disk-cache.cc

@@ -166,16 +166,37 @@ public:
         return i->second;
     }
 
+    std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
+    {
+        auto i = state.caches.find(uri);
+        if (i == state.caches.end()) {
+            auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
+            if (!queryCache.next())
+                return std::nullopt;
+            state.caches.emplace(uri,
+                Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
+        }
+        return getCache(state, uri);
+    }
+
     void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
     {
         retrySQLite<void>([&]() {
             auto state(_state.lock());
+            SQLiteTxn txn(state->db);
 
-            // FIXME: race
+            // To avoid the race, we have to check if maybe someone hasn't yet created
+            // the cache for this URI in the meantime.
+            auto cache(queryCacheRaw(*state, uri));
+
+            if (cache)
+                return;
 
             state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
             assert(sqlite3_changes(state->db) == 1);
             state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+
+            txn.commit();
         });
     }
 
@@ -183,21 +204,12 @@ public:
     {
         return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
             auto state(_state.lock());
-
+            auto cache(queryCacheRaw(*state, uri));
-            auto i = state->caches.find(uri);
+            if (!cache)
-            if (i == state->caches.end()) {
-                auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl));
-                if (!queryCache.next())
                 return std::nullopt;
-                state->caches.emplace(uri,
-                    Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)});
-            }
-
-            auto & cache(getCache(*state, uri));
-
             return CacheInfo {
-                .wantMassQuery = cache.wantMassQuery,
+                .wantMassQuery = cache->wantMassQuery,
-                .priority = cache.priority
+                .priority = cache->priority
             };
         });
     }

src/libstore/sqlite.cc

@@ -8,12 +8,15 @@
 
 namespace nix {
 
-SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf)
+SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf)
-  : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo)
+  : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset)
 {
-    err.msg = hintfmt("%s: %s (in '%s')",
+    auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": ";
+    err.msg = hintfmt("%s: %s%s, %s (in '%s')",
         normaltxt(hf.str()),
+        offsetStr,
         sqlite3_errstr(extendedErrNo),
+        errMsg,
         path ? path : "(in-memory)");
 }
 
@@ -21,11 +24,13 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor
 {
     int err = sqlite3_errcode(db);
     int exterr = sqlite3_extended_errcode(db);
+    int offset = sqlite3_error_offset(db);
 
     auto path = sqlite3_db_filename(db, nullptr);
+    auto errMsg = sqlite3_errmsg(db);
 
     if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) {
-        auto exp = SQLiteBusy(path, err, exterr, std::move(hf));
+        auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf));
         exp.err.msg = hintfmt(
             err == SQLITE_PROTOCOL
                 ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)"
@@ -33,7 +38,7 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor
             path ? path : "(in-memory)");
         throw exp;
     } else
-        throw SQLiteError(path, err, exterr, std::move(hf));
+        throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf));
 }
 
 SQLite::SQLite(const Path & path, bool create)

src/libstore/sqlite.hh

@@ -98,21 +98,22 @@ struct SQLiteTxn
 
 struct SQLiteError : Error
 {
-    const char *path;
+    std::string path;
-    int errNo, extendedErrNo;
+    std::string errMsg;
+    int errNo, extendedErrNo, offset;
 
     template<typename... Args>
     [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) {
         throw_(db, hintfmt(fs, args...));
     }
 
-    SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf);
+    SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf);
 
 protected:
 
     template<typename... Args>
-    SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args)
+    SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args)
-      : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...))
+      : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...))
     { }
 
     [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf);

src/libutil/canon-path.cc

@@ -0,0 +1,103 @@
+#include "canon-path.hh"
+#include "util.hh"
+
+namespace nix {
+
+CanonPath CanonPath::root = CanonPath("/");
+
+CanonPath::CanonPath(std::string_view raw)
+    : path(absPath((Path) raw, "/"))
+{ }
+
+CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
+    : path(absPath((Path) raw, root.abs()))
+{ }
+
+std::optional<CanonPath> CanonPath::parent() const
+{
+    if (isRoot()) return std::nullopt;
+    return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
+}
+
+void CanonPath::pop()
+{
+    assert(!isRoot());
+    path.resize(std::max((size_t) 1, path.rfind('/')));
+}
+
+bool CanonPath::isWithin(const CanonPath & parent) const
+{
+    return !(
+        path.size() < parent.path.size()
+        || path.substr(0, parent.path.size()) != parent.path
+        || (parent.path.size() > 1 && path.size() > parent.path.size()
+            && path[parent.path.size()] != '/'));
+}
+
+CanonPath CanonPath::removePrefix(const CanonPath & prefix) const
+{
+    assert(isWithin(prefix));
+    if (prefix.isRoot()) return *this;
+    if (path.size() == prefix.path.size()) return root;
+    return CanonPath(unchecked_t(), path.substr(prefix.path.size()));
+}
+
+void CanonPath::extend(const CanonPath & x)
+{
+    if (x.isRoot()) return;
+    if (isRoot())
+        path += x.rel();
+    else
+        path += x.abs();
+}
+
+CanonPath CanonPath::operator + (const CanonPath & x) const
+{
+    auto res = *this;
+    res.extend(x);
+    return res;
+}
+
+void CanonPath::push(std::string_view c)
+{
+    assert(c.find('/') == c.npos);
+    assert(c != "." && c != "..");
+    if (!isRoot()) path += '/';
+    path += c;
+}
+
+CanonPath CanonPath::operator + (std::string_view c) const
+{
+    auto res = *this;
+    res.push(c);
+    return res;
+}
+
+bool CanonPath::isAllowed(const std::set<CanonPath> & allowed) const
+{
+    /* Check if `this` is an exact match or the parent of an
+       allowed path. */
+    auto lb = allowed.lower_bound(*this);
+    if (lb != allowed.end()) {
+        if (lb->isWithin(*this))
+            return true;
+    }
+
+    /* Check if a parent of `this` is allowed. */
+    auto path = *this;
+    while (!path.isRoot()) {
+        path.pop();
+        if (allowed.count(path))
+            return true;
+    }
+
+    return false;
+}
+
+std::ostream & operator << (std::ostream & stream, const CanonPath & path)
+{
+    stream << path.abs();
+    return stream;
+}
+
+}

src/libutil/canon-path.hh

@@ -0,0 +1,173 @@
+#pragma once
+
+#include <string>
+#include <optional>
+#include <cassert>
+#include <iostream>
+#include <set>
+
+namespace nix {
+
+/* A canonical representation of a path. It ensures the following:
+
+   - It always starts with a slash.
+
+   - It never ends with a slash, except if the path is "/".
+
+   - A slash is never followed by a slash (i.e. no empty components).
+
+   - There are no components equal to '.' or '..'.
+
+   Note that the path does not need to correspond to an actually
+   existing path, and there is no guarantee that symlinks are
+   resolved.
+*/
+class CanonPath
+{
+    std::string path;
+
+public:
+
+    /* Construct a canon path from a non-canonical path. Any '.', '..'
+       or empty components are removed. */
+    CanonPath(std::string_view raw);
+
+    explicit CanonPath(const char * raw)
+        : CanonPath(std::string_view(raw))
+    { }
+
+    struct unchecked_t { };
+
+    CanonPath(unchecked_t _, std::string path)
+        : path(std::move(path))
+    { }
+
+    static CanonPath root;
+
+    /* If `raw` starts with a slash, return
+       `CanonPath(raw)`. Otherwise return a `CanonPath` representing
+       `root + "/" + raw`. */
+    CanonPath(std::string_view raw, const CanonPath & root);
+
+    bool isRoot() const
+    { return path.size() <= 1; }
+
+    explicit operator std::string_view() const
+    { return path; }
+
+    const std::string & abs() const
+    { return path; }
+
+    /* Like abs(), but return an empty string if this path is
+       '/'. Thus the returned string never ends in a slash. */
+    const std::string & absOrEmpty() const
+    {
+        const static std::string epsilon;
+        return isRoot() ? epsilon : path;
+    }
+
+    const char * c_str() const
+    { return path.c_str(); }
+
+    std::string_view rel() const
+    { return ((std::string_view) path).substr(1); }
+
+    struct Iterator
+    {
+        std::string_view remaining;
+        size_t slash;
+
+        Iterator(std::string_view remaining)
+            : remaining(remaining)
+            , slash(remaining.find('/'))
+        { }
+
+        bool operator != (const Iterator & x) const
+        { return remaining.data() != x.remaining.data(); }
+
+        const std::string_view operator * () const
+        { return remaining.substr(0, slash); }
+
+        void operator ++ ()
+        {
+            if (slash == remaining.npos)
+                remaining = remaining.substr(remaining.size());
+            else {
+                remaining = remaining.substr(slash + 1);
+                slash = remaining.find('/');
+            }
+        }
+    };
+
+    Iterator begin() const { return Iterator(rel()); }
+    Iterator end() const { return Iterator(rel().substr(path.size() - 1)); }
+
+    std::optional<CanonPath> parent() const;
+
+    /* Remove the last component. Panics if this path is the root.  */
+    void pop();
+
+    std::optional<std::string_view> dirOf() const
+    {
+        if (isRoot()) return std::nullopt;
+        return ((std::string_view) path).substr(0, path.rfind('/'));
+    }
+
+    std::optional<std::string_view> baseName() const
+    {
+        if (isRoot()) return std::nullopt;
+        return ((std::string_view) path).substr(path.rfind('/') + 1);
+    }
+
+    bool operator == (const CanonPath & x) const
+    { return path == x.path; }
+
+    bool operator != (const CanonPath & x) const
+    { return path != x.path; }
+
+    /* Compare paths lexicographically except that path separators
+       are sorted before any other character. That is, in the sorted order
+       a directory is always followed directly by its children. For
+       instance, 'foo' < 'foo/bar' < 'foo!'. */
+    bool operator < (const CanonPath & x) const
+    {
+        auto i = path.begin();
+        auto j = x.path.begin();
+        for ( ; i != path.end() && j != x.path.end(); ++i, ++j) {
+            auto c_i = *i;
+            if (c_i == '/') c_i = 0;
+            auto c_j = *j;
+            if (c_j == '/') c_j = 0;
+            if (c_i < c_j) return true;
+            if (c_i > c_j) return false;
+        }
+        return i == path.end() && j != x.path.end();
+    }
+
+    /* Return true if `this` is equal to `parent` or a child of
+       `parent`. */
+    bool isWithin(const CanonPath & parent) const;
+
+    CanonPath removePrefix(const CanonPath & prefix) const;
+
+    /* Append another path to this one. */
+    void extend(const CanonPath & x);
+
+    /* Concatenate two paths. */
+    CanonPath operator + (const CanonPath & x) const;
+
+    /* Add a path component to this one. It must not contain any slashes. */
+    void push(std::string_view c);
+
+    CanonPath operator + (std::string_view c) const;
+
+    /* Check whether access to this path is allowed, which is the case
+       if 1) `this` is within any of the `allowed` paths; or 2) any of
+       the `allowed` paths are within `this`. (The latter condition
+       ensures access to the parents of allowed paths.) */
+    bool isAllowed(const std::set<CanonPath> & allowed) const;
+};
+
+std::ostream & operator << (std::ostream & stream, const CanonPath & path);
+
+}

src/libutil/error.cc

@@ -9,9 +9,9 @@ namespace nix {
 
 const std::string nativeSystem = SYSTEM;
 
-void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
+void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint)
 {
-    err.traces.push_front(Trace { .pos = e, .hint = hint });
+    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint });
 }
 
 // c++ std::exception descendants must have a 'const char* what()' function.
@@ -35,86 +35,41 @@ std::ostream & operator<<(std::ostream & os, const hintformat & hf)
     return os << hf.str();
 }
 
-std::string showErrPos(const ErrPos & errPos)
+std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
 {
-    if (errPos.line > 0) {
+    pos.print(str);
-        if (errPos.column > 0) {
+    str << ":" << pos.line;
-            return fmt("%d:%d", errPos.line, errPos.column);
+    if (pos.column > 0)
-        } else {
+        str << ":" << pos.column;
-            return fmt("%d", errPos.line);
+    return str;
-        }
-    }
-    else {
-        return "";
-    }
 }
 
-std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
+std::optional<LinesOfCode> AbstractPos::getCodeLines() const
 {
-    if (errPos.line <= 0)
+    if (line == 0)
         return std::nullopt;
 
-    if (errPos.origin == foFile) {
+    if (auto source = getSource()) {
-        LinesOfCode loc;
-        try {
-            // FIXME: when running as the daemon, make sure we don't
-            // open a file to which the client doesn't have access.
-            AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
-            if (!fd) return {};
 
+        std::istringstream iss(*source);
         // count the newlines.
         int count = 0;
-            std::string line;
+        std::string curLine;
-            int pl = errPos.line - 1;
+        int pl = line - 1;
-            do
-            {
-                line = readLine(fd.get());
-                ++count;
-                if (count < pl)
-                    ;
-                else if (count == pl)
-                    loc.prevLineOfCode = line;
-                else if (count == pl + 1)
-                    loc.errLineOfCode = line;
-                else if (count == pl + 2) {
-                    loc.nextLineOfCode = line;
-                    break;
-                }
-            } while (true);
-            return loc;
-        }
-        catch (EndOfFile & eof) {
-            if (loc.errLineOfCode.has_value())
-                return loc;
-            else
-                return std::nullopt;
-        }
-        catch (std::exception & e) {
-            return std::nullopt;
-        }
-    } else {
-        std::istringstream iss(errPos.file);
-        // count the newlines.
-        int count = 0;
-        std::string line;
-        int pl = errPos.line - 1;
 
         LinesOfCode loc;
 
-        do
+        do {
-        {
+            std::getline(iss, curLine);
-            std::getline(iss, line);
             ++count;
             if (count < pl)
-            {
                 ;
-            }
             else if (count == pl) {
-                loc.prevLineOfCode = line;
+                loc.prevLineOfCode = curLine;
             } else if (count == pl + 1) {
-                loc.errLineOfCode = line;
+                loc.errLineOfCode = curLine;
             } else if (count == pl + 2) {
-                loc.nextLineOfCode = line;
+                loc.nextLineOfCode = curLine;
                 break;
             }
 
@@ -124,12 +79,14 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
 
         return loc;
     }
+
+    return std::nullopt;
 }
 
 // print lines of code to the ostream, indicating the error column.
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const ErrPos & errPos,
+    const AbstractPos & errPos,
     const LinesOfCode & loc)
 {
     // previous line of code.
@@ -176,28 +133,6 @@ void printCodeLines(std::ostream & out,
     }
 }
 
-void printAtPos(const ErrPos & pos, std::ostream & out)
-{
-    if (pos) {
-        switch (pos.origin) {
-            case foFile: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
-                break;
-            }
-            case foString: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
-                break;
-            }
-            case foStdin: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
-                break;
-            }
-            default:
-                throw Error("invalid FileOrigin in errPos");
-        }
-    }
-}
-
 static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s)
 {
     std::string res;
@@ -263,22 +198,22 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     std::ostringstream oss;
 
+    auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n";
+
     // traces
     if (showTrace && !einfo.traces.empty()) {
         for (const auto & trace : einfo.traces) {
             oss << "\n" << "… " << trace.hint.str() << "\n";
 
-            if (trace.pos.has_value() && (*trace.pos)) {
+            if (trace.pos) {
-                auto pos = trace.pos.value();
+                oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":";
-                oss << "\n";
-                printAtPos(pos, oss);
 
-                auto loc = getCodeLines(pos);
+                if (auto loc = trace.pos->getCodeLines()) {
-                if (loc.has_value()) {
                     oss << "\n";
-                    printCodeLines(oss, "", pos, *loc);
+                    printCodeLines(oss, "", *trace.pos, *loc);
                     oss << "\n";
-                }
+                } else
+                    oss << noSource;
             }
         }
         oss << "\n" << prefix;
@@ -286,18 +221,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
 
     oss << einfo.msg << "\n";
 
-    if (einfo.errPos.has_value() && *einfo.errPos) {
+    if (einfo.errPos) {
-        oss << "\n";
+        oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":";
-        printAtPos(*einfo.errPos, oss);
 
-        auto loc = getCodeLines(*einfo.errPos);
+        if (auto loc = einfo.errPos->getCodeLines()) {
-
-        // lines of code.
-        if (loc.has_value()) {
             oss << "\n";
             printCodeLines(oss, "", *einfo.errPos, *loc);
             oss << "\n";
-        }
+        } else
+            oss << noSource;
     }
 
     auto suggestions = einfo.suggestions.trim();

src/libutil/error.hh

@@ -54,13 +54,6 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
-/* adjust Pos::origin bit width when adding stuff here */
-typedef enum {
-    foFile,
-    foStdin,
-    foString
-} FileOrigin;
-
 // the lines of code surrounding an error.
 struct LinesOfCode {
     std::optional<std::string> prevLineOfCode;
@@ -68,54 +61,37 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
-// ErrPos indicates the location of an error in a nix file.
+/* An abstract type that represents a location in a source file. */
-struct ErrPos {
+struct AbstractPos
-    int line = 0;
-    int column = 0;
-    std::string file;
-    FileOrigin origin;
-
-    operator bool() const
 {
-        return line != 0;
+    uint32_t line = 0;
-    }
+    uint32_t column = 0;
 
-    // convert from the Pos struct, found in libexpr.
+    /* Return the contents of the source file. */
-    template <class P>
+    virtual std::optional<std::string> getSource() const
-    ErrPos & operator=(const P & pos)
+    { return std::nullopt; };
-    {
-        origin = pos.origin;
-        line = pos.line;
-        column = pos.column;
-        file = pos.file;
-        return *this;
-    }
 
-    template <class P>
+    virtual void print(std::ostream & out) const = 0;
-    ErrPos(const P & p)
+
-    {
+    std::optional<LinesOfCode> getCodeLines() const;
-        *this = p;
-    }
 };
 
-std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos);
+std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
 
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const ErrPos & errPos,
+    const AbstractPos & errPos,
     const LinesOfCode & loc);
 
-void printAtPos(const ErrPos & pos, std::ostream & out);
-
 struct Trace {
-    std::optional<ErrPos> pos;
+    std::shared_ptr<AbstractPos> pos;
     hintformat hint;
 };
 
 struct ErrorInfo {
     Verbosity level;
     hintformat msg;
-    std::optional<ErrPos> errPos;
+    std::shared_ptr<AbstractPos> errPos;
     std::list<Trace> traces;
 
     Suggestions suggestions;
@@ -177,12 +153,12 @@ public:
     const ErrorInfo & info() const { calcWhat(); return err; }
 
     template<typename... Args>
-    void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
+    void addTrace(std::shared_ptr<AbstractPos> && e, const std::string & fs, const Args & ... args)
     {
-        addTrace(e, hintfmt(fs, args...));
+        addTrace(std::move(e), hintfmt(fs, args...));
     }
 
-    void addTrace(std::optional<ErrPos> e, hintformat hint);
+    void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint);
 
     bool hasTrace() const { return !err.traces.empty(); }
 };

src/libutil/logging.cc

@@ -131,6 +131,21 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
+void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos)
+{
+    if (pos) {
+        json["line"] = pos->line;
+        json["column"] = pos->column;
+        std::ostringstream str;
+        pos->print(str);
+        json["file"] = str.str();
+    } else {
+        json["line"] = nullptr;
+        json["column"] = nullptr;
+        json["file"] = nullptr;
+    }
+}
+
 struct JSONLogger : Logger {
     Logger & prevLogger;
 
@@ -177,27 +192,14 @@ struct JSONLogger : Logger {
         json["level"] = ei.level;
         json["msg"] = oss.str();
         json["raw_msg"] = ei.msg.str();
-
+        to_json(json, ei.errPos);
-        if (ei.errPos.has_value() && (*ei.errPos)) {
-            json["line"] = ei.errPos->line;
-            json["column"] = ei.errPos->column;
-            json["file"] = ei.errPos->file;
-        } else {
-            json["line"] = nullptr;
-            json["column"] = nullptr;
-            json["file"] = nullptr;
-        }
 
         if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
             nlohmann::json traces = nlohmann::json::array();
             for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
                 nlohmann::json stackFrame;
                 stackFrame["raw_msg"] = iter->hint.str();
-                if (iter->pos.has_value() && (*iter->pos)) {
+                to_json(stackFrame, iter->pos);
-                    stackFrame["line"] = iter->pos->line;
-                    stackFrame["column"] = iter->pos->column;
-                    stackFrame["file"] = iter->pos->file;
-                }
                 traces.push_back(stackFrame);
             }
 

src/libutil/tests/canon-path.cc

@@ -0,0 +1,155 @@
+#include "canon-path.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    TEST(CanonPath, basic) {
+        {
+            CanonPath p("/");
+            ASSERT_EQ(p.abs(), "/");
+            ASSERT_EQ(p.rel(), "");
+            ASSERT_EQ(p.baseName(), std::nullopt);
+            ASSERT_EQ(p.dirOf(), std::nullopt);
+            ASSERT_FALSE(p.parent());
+        }
+
+        {
+            CanonPath p("/foo//");
+            ASSERT_EQ(p.abs(), "/foo");
+            ASSERT_EQ(p.rel(), "foo");
+            ASSERT_EQ(*p.baseName(), "foo");
+            ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this?
+            ASSERT_EQ(p.parent()->abs(), "/");
+        }
+
+        {
+            CanonPath p("foo/bar");
+            ASSERT_EQ(p.abs(), "/foo/bar");
+            ASSERT_EQ(p.rel(), "foo/bar");
+            ASSERT_EQ(*p.baseName(), "bar");
+            ASSERT_EQ(*p.dirOf(), "/foo");
+            ASSERT_EQ(p.parent()->abs(), "/foo");
+        }
+
+        {
+            CanonPath p("foo//bar/");
+            ASSERT_EQ(p.abs(), "/foo/bar");
+            ASSERT_EQ(p.rel(), "foo/bar");
+            ASSERT_EQ(*p.baseName(), "bar");
+            ASSERT_EQ(*p.dirOf(), "/foo");
+        }
+    }
+
+    TEST(CanonPath, pop) {
+        CanonPath p("foo/bar/x");
+        ASSERT_EQ(p.abs(), "/foo/bar/x");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/foo/bar");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/foo");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/");
+    }
+
+    TEST(CanonPath, removePrefix) {
+        CanonPath p1("foo/bar");
+        CanonPath p2("foo/bar/a/b/c");
+        ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c");
+        ASSERT_EQ(p1.removePrefix(p1).abs(), "/");
+        ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar");
+    }
+
+    TEST(CanonPath, iter) {
+        {
+            CanonPath p("a//foo/bar//");
+            std::vector<std::string_view> ss;
+            for (auto & c : p) ss.push_back(c);
+            ASSERT_EQ(ss, std::vector<std::string_view>({"a", "foo", "bar"}));
+        }
+
+        {
+            CanonPath p("/");
+            std::vector<std::string_view> ss;
+            for (auto & c : p) ss.push_back(c);
+            ASSERT_EQ(ss, std::vector<std::string_view>());
+        }
+    }
+
+    TEST(CanonPath, concat) {
+        {
+            CanonPath p1("a//foo/bar//");
+            CanonPath p2("xyzzy/bla");
+            ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla");
+        }
+
+        {
+            CanonPath p1("/");
+            CanonPath p2("/a/b");
+            ASSERT_EQ((p1 + p2).abs(), "/a/b");
+        }
+
+        {
+            CanonPath p1("/a/b");
+            CanonPath p2("/");
+            ASSERT_EQ((p1 + p2).abs(), "/a/b");
+        }
+
+        {
+            CanonPath p("/foo/bar");
+            ASSERT_EQ((p + "x").abs(), "/foo/bar/x");
+        }
+
+        {
+            CanonPath p("/");
+            ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar");
+        }
+    }
+
+    TEST(CanonPath, within) {
+        {
+            ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo")));
+            ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar")));
+            ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/")));
+            ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/")));
+        }
+    }
+
+    TEST(CanonPath, sort) {
+        ASSERT_FALSE(CanonPath("foo") < CanonPath("foo"));
+        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar"));
+        ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!"));
+        ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo"));
+        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!"));
+    }
+
+    TEST(CanonPath, allowed) {
+        {
+            std::set<CanonPath> allowed {
+                CanonPath("foo/bar"),
+                CanonPath("foo!"),
+                CanonPath("xyzzy"),
+                CanonPath("a/b/c"),
+            };
+
+            ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("foo").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("bar").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("/").isAllowed(allowed));
+        }
+    }
+}

src/nix-env/nix-env.cc

@@ -647,7 +647,7 @@ static void upgradeDerivations(Globals & globals,
                 } else newElems.push_back(i);
 
             } catch (Error & e) {
-                e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName());
+                e.addTrace(nullptr, "while trying to find an upgrade for '%s'", i.queryName());
                 throw;
             }
         }
@@ -958,7 +958,7 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
         } catch (AssertionError & e) {
             printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
+            e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName());
             throw;
         }
     }
@@ -1262,7 +1262,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
         } catch (AssertionError & e) {
             printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
         } catch (Error & e) {
-            e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
+            e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName());
             throw;
         }
     }

src/nix/develop.cc

@@ -192,10 +192,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     drv.env.erase("allowedRequisites");
     drv.env.erase("disallowedReferences");
     drv.env.erase("disallowedRequisites");
+    drv.env.erase("name");
 
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
+    drv.env.emplace("name", drv.name);
     drv.inputSrcs.insert(std::move(getEnvShPath));
     if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
         for (auto & output : drv.outputs) {

tests/fetchGitSubmodules.sh

@@ -14,6 +14,15 @@ subRepo=$TEST_ROOT/gitSubmodulesSub
 
 rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix
 
+# Submodules can't be fetched locally by default, which can cause
+# information leakage vulnerabilities, but for these tests our
+# submodule is intentionally local and it's all trusted, so we
+# disable this restriction. Setting it per repo is not sufficient, as
+# the repo-local config does not apply to the commands run from
+# outside the repos by Nix.
+export XDG_CONFIG_HOME=$TEST_HOME/.config
+git config --global protocol.file.allow always
+
 initGitRepo() {
     git init $1
     git -C $1 config user.email "foobar@example.com"

tests/function-trace.sh

@@ -32,40 +32,40 @@ expect_trace() {
 
 # failure inside a tryEval
 expect_trace 'builtins.tryEval (throw "example")' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace entered (string):1:19 at
+function-trace entered «string»:1:19 at
-function-trace exited (string):1:19 at
+function-trace exited «string»:1:19 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 # Missing argument to a formal function
 expect_trace '({ x }: x) { }' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 # Too many arguments to a formal function
 expect_trace '({ x }: x) { x = "x"; y = "y"; }' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 # Not enough arguments to a lambda
 expect_trace '(x: y: x + y) 1' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 # Too many arguments to a lambda
 expect_trace '(x: x) 1 2' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 # Not a function
 expect_trace '1 2' "
-function-trace entered (string):1:1 at
+function-trace entered «string»:1:1 at
-function-trace exited (string):1:1 at
+function-trace exited «string»:1:1 at
 "
 
 set -e