Merge remote-tracking branch 'upstream/master' into overlayfs-store

.github/workflows/labels.yml

@@ -21,4 +21,4 @@ jobs:
     - uses: actions/labeler@v4
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        sync-labels: true
+        sync-labels: false

src/libcmd/common-eval-args.cc

@@ -1,3 +1,4 @@
+#include "eval-settings.hh"
 #include "common-eval-args.hh"
 #include "shared.hh"
 #include "filetransfer.hh"

src/libcmd/installables.cc

@@ -11,6 +11,7 @@
 #include "derivations.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "get-drvs.hh"
 #include "store-api.hh"
 #include "shared.hh"

src/libcmd/repl.cc

@@ -26,6 +26,7 @@ extern "C" {
 #include "eval.hh"
 #include "eval-cache.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "attr-path.hh"
 #include "store-api.hh"
 #include "log-store.hh"

src/libexpr/eval-settings.cc

@@ -0,0 +1,95 @@
+#include "globals.hh"
+#include "profiles.hh"
+#include "eval.hh"
+#include "eval-settings.hh"
+
+namespace nix {
+
+/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
+   can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
+static Strings parseNixPath(const std::string & s)
+{
+    Strings res;
+
+    auto p = s.begin();
+
+    while (p != s.end()) {
+        auto start = p;
+        auto start2 = p;
+
+        while (p != s.end() && *p != ':') {
+            if (*p == '=') start2 = p + 1;
+            ++p;
+        }
+
+        if (p == s.end()) {
+            if (p != start) res.push_back(std::string(start, p));
+            break;
+        }
+
+        if (*p == ':') {
+            auto prefix = std::string(start2, s.end());
+            if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
+                ++p;
+                while (p != s.end() && *p != ':') ++p;
+            }
+            res.push_back(std::string(start, p));
+            if (p == s.end()) break;
+        }
+
+        ++p;
+    }
+
+    return res;
+}
+
+EvalSettings::EvalSettings()
+{
+    auto var = getEnv("NIX_PATH");
+    if (var) nixPath = parseNixPath(*var);
+}
+
+Strings EvalSettings::getDefaultNixPath()
+{
+    Strings res;
+    auto add = [&](const Path & p, const std::string & s = std::string()) {
+        if (pathAccessible(p)) {
+            if (s.empty()) {
+                res.push_back(p);
+            } else {
+                res.push_back(s + "=" + p);
+            }
+        }
+    };
+
+    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
+        add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
+        add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
+        add(rootChannelsDir());
+    }
+
+    return res;
+}
+
+bool EvalSettings::isPseudoUrl(std::string_view s)
+{
+    if (s.compare(0, 8, "channel:") == 0) return true;
+    size_t pos = s.find("://");
+    if (pos == std::string::npos) return false;
+    std::string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
+}
+
+std::string EvalSettings::resolvePseudoUrl(std::string_view url)
+{
+    if (hasPrefix(url, "channel:"))
+        return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
+    else
+        return std::string(url);
+}
+
+EvalSettings evalSettings;
+
+static GlobalConfig::Register rEvalSettings(&evalSettings);
+
+}

src/libexpr/eval-settings.hh

@@ -0,0 +1,98 @@
+#pragma once
+#include "config.hh"
+
+namespace nix {
+
+struct EvalSettings : Config
+{
+    EvalSettings();
+
+    static Strings getDefaultNixPath();
+
+    static bool isPseudoUrl(std::string_view s);
+
+    static std::string resolvePseudoUrl(std::string_view url);
+
+    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
+        "Whether builtin functions that allow executing native code should be enabled."};
+
+    Setting<Strings> nixPath{
+        this, getDefaultNixPath(), "nix-path",
+        R"(
+          List of directories to be searched for `<...>` file references
+
+          In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
+          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
+        )"};
+
+    Setting<bool> restrictEval{
+        this, false, "restrict-eval",
+        R"(
+          If set to `true`, the Nix evaluator will not allow access to any
+          files outside of the Nix search path (as set via the `NIX_PATH`
+          environment variable or the `-I` option), or to URIs outside of
+          [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
+          The default is `false`.
+        )"};
+
+    Setting<bool> pureEval{this, false, "pure-eval",
+        R"(
+          Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
+
+          - Restrict file system and network access to files specified by cryptographic hash
+          - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
+        )"
+        };
+
+    Setting<bool> enableImportFromDerivation{
+        this, true, "allow-import-from-derivation",
+        R"(
+          By default, Nix allows you to `import` from a derivation, allowing
+          building at evaluation time. With this option set to false, Nix will
+          throw an error when evaluating an expression that uses this feature,
+          allowing users to ensure their evaluation will not require any
+          builds to take place.
+        )"};
+
+    Setting<Strings> allowedUris{this, {}, "allowed-uris",
+        R"(
+          A list of URI prefixes to which access is allowed in restricted
+          evaluation mode. For example, when set to
+          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
+          allowed to access `https://github.com/NixOS/patchelf.git`.
+        )"};
+
+    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
+        R"(
+          If set to `true`, the Nix evaluator will trace every function call.
+          Nix will print a log message at the "vomit" level for every function
+          entrance and function exit.
+
+              function-trace entered undefined position at 1565795816999559622
+              function-trace exited undefined position at 1565795816999581277
+              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
+              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
+
+          The `undefined position` means the function call is a builtin.
+
+          Use the `contrib/stack-collapse.py` script distributed with the Nix
+          source code to convert the trace logs in to a format suitable for
+          `flamegraph.pl`.
+        )"};
+
+    Setting<bool> useEvalCache{this, true, "eval-cache",
+        "Whether to use the flake evaluation cache."};
+
+    Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
+        R"(
+          If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
+          debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
+        )"};
+
+    Setting<bool> traceVerbose{this, false, "trace-verbose",
+        "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
+};
+
+extern EvalSettings evalSettings;
+
+}

src/libexpr/eval.cc

@@ -1,4 +1,5 @@
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "hash.hh"
 #include "types.hh"
 #include "util.hh"
@@ -420,44 +421,6 @@ void initGC()
 }
 
 
-/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
-   can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
-static Strings parseNixPath(const std::string & s)
-{
-    Strings res;
-
-    auto p = s.begin();
-
-    while (p != s.end()) {
-        auto start = p;
-        auto start2 = p;
-
-        while (p != s.end() && *p != ':') {
-            if (*p == '=') start2 = p + 1;
-            ++p;
-        }
-
-        if (p == s.end()) {
-            if (p != start) res.push_back(std::string(start, p));
-            break;
-        }
-
-        if (*p == ':') {
-            auto prefix = std::string(start2, s.end());
-            if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
-                ++p;
-                while (p != s.end() && *p != ':') ++p;
-            }
-            res.push_back(std::string(start, p));
-            if (p == s.end()) break;
-        }
-
-        ++p;
-    }
-
-    return res;
-}
-
 ErrorBuilder & ErrorBuilder::atPos(PosIdx pos)
 {
     info.errPos = state.positions[pos];
@@ -2626,54 +2589,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
 }
 
 
-EvalSettings::EvalSettings()
-{
-    auto var = getEnv("NIX_PATH");
-    if (var) nixPath = parseNixPath(*var);
-}
-
-Strings EvalSettings::getDefaultNixPath()
-{
-    Strings res;
-    auto add = [&](const Path & p, const std::string & s = std::string()) {
-        if (pathAccessible(p)) {
-            if (s.empty()) {
-                res.push_back(p);
-            } else {
-                res.push_back(s + "=" + p);
-            }
-        }
-    };
-
-    if (!evalSettings.restrictEval && !evalSettings.pureEval) {
-        add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
-        add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
-        add(rootChannelsDir());
-    }
-
-    return res;
-}
-
-bool EvalSettings::isPseudoUrl(std::string_view s)
-{
-    if (s.compare(0, 8, "channel:") == 0) return true;
-    size_t pos = s.find("://");
-    if (pos == std::string::npos) return false;
-    std::string scheme(s, 0, pos);
-    return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
-}
-
-std::string EvalSettings::resolvePseudoUrl(std::string_view url)
-{
-    if (hasPrefix(url, "channel:"))
-        return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
-    else
-        return std::string(url);
-}
-
-EvalSettings evalSettings;
-
-static GlobalConfig::Register rEvalSettings(&evalSettings);
-
-
 }

src/libexpr/eval.hh

@@ -787,98 +787,6 @@ struct InvalidPathError : EvalError
 #endif
 };
 
-struct EvalSettings : Config
-{
-    EvalSettings();
-
-    static Strings getDefaultNixPath();
-
-    static bool isPseudoUrl(std::string_view s);
-
-    static std::string resolvePseudoUrl(std::string_view url);
-
-    Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
-        "Whether builtin functions that allow executing native code should be enabled."};
-
-    Setting<Strings> nixPath{
-        this, getDefaultNixPath(), "nix-path",
-        R"(
-          List of directories to be searched for `<...>` file references
-
-          In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
-          [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
-        )"};
-
-    Setting<bool> restrictEval{
-        this, false, "restrict-eval",
-        R"(
-          If set to `true`, the Nix evaluator will not allow access to any
-          files outside of the Nix search path (as set via the `NIX_PATH`
-          environment variable or the `-I` option), or to URIs outside of
-          [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
-          The default is `false`.
-        )"};
-
-    Setting<bool> pureEval{this, false, "pure-eval",
-        R"(
-          Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
-
-          - Restrict file system and network access to files specified by cryptographic hash
-          - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
-        )"
-        };
-
-    Setting<bool> enableImportFromDerivation{
-        this, true, "allow-import-from-derivation",
-        R"(
-          By default, Nix allows you to `import` from a derivation, allowing
-          building at evaluation time. With this option set to false, Nix will
-          throw an error when evaluating an expression that uses this feature,
-          allowing users to ensure their evaluation will not require any
-          builds to take place.
-        )"};
-
-    Setting<Strings> allowedUris{this, {}, "allowed-uris",
-        R"(
-          A list of URI prefixes to which access is allowed in restricted
-          evaluation mode. For example, when set to
-          `https://github.com/NixOS`, builtin functions such as `fetchGit` are
-          allowed to access `https://github.com/NixOS/patchelf.git`.
-        )"};
-
-    Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
-        R"(
-          If set to `true`, the Nix evaluator will trace every function call.
-          Nix will print a log message at the "vomit" level for every function
-          entrance and function exit.
-
-              function-trace entered undefined position at 1565795816999559622
-              function-trace exited undefined position at 1565795816999581277
-              function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
-              function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
-
-          The `undefined position` means the function call is a builtin.
-
-          Use the `contrib/stack-collapse.py` script distributed with the Nix
-          source code to convert the trace logs in to a format suitable for
-          `flamegraph.pl`.
-        )"};
-
-    Setting<bool> useEvalCache{this, true, "eval-cache",
-        "Whether to use the flake evaluation cache."};
-
-    Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
-        R"(
-          If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
-          debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
-        )"};
-
-    Setting<bool> traceVerbose{this, false, "trace-verbose",
-        "Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
-};
-
-extern EvalSettings evalSettings;
-
 static const std::string corepkgsPrefix{"/__corepkgs__/"};
 
 template<class ErrorType>

src/libexpr/flake/flake.cc

@@ -1,5 +1,6 @@
 #include "flake.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "lockfile.hh"
 #include "primops.hh"
 #include "eval-inline.hh"

src/libexpr/parser.y

@@ -22,6 +22,7 @@
 
 #include "nixexpr.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "globals.hh"
 
 namespace nix {

src/libexpr/primops.cc

@@ -3,6 +3,7 @@
 #include "downstream-placeholder.hh"
 #include "eval-inline.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"

src/libexpr/primops/fetchMercurial.cc

@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "url.hh"

src/libexpr/primops/fetchTree.cc

@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "store-api.hh"
 #include "fetchers.hh"
 #include "filetransfer.hh"

src/libstore/local-store.cc

@@ -1593,8 +1593,21 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
 
 bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths)
 {
-    StringSet store;
-    for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
+    StorePathSet storePathsInStoreDir;
+    /* Why aren't we using `queryAllValidPaths`? Because that would
+       tell us about all the paths than the database knows about. Here we
+       want to know about all the store paths in the store directory,
+       regardless of what the database thinks.
+
+       We will end up cross-referencing these two sources of truth (the
+       database and the filesystem) in the loop below, in order to catch
+       invalid states.
+     */
+    for (auto & i : readDirectory(realStoreDir)) {
+        try {
+            storePathsInStoreDir.insert({i.name});
+        } catch (BadStorePath &) { }
+    }
 
     /* Check whether all valid paths actually exist. */
     printInfo("checking path existence...");
@@ -1603,7 +1616,7 @@ bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPath
     bool errors = false;
 
     auto existsInStoreDir = [&](const StorePath & storePath) {
-        return store.count(std::string(storePath.to_string()));
+        return storePathsInStoreDir.count(storePath);
     };
 
     for (auto & i : queryAllValidPaths())
@@ -1620,8 +1633,6 @@ void LocalStore::verifyPath(const StorePath & path, std::function<bool(const Sto
 
     if (!done.insert(path).second) return;
 
-    auto pathS = printStorePath(path);
-
     if (!existsInStoreDir(path)) {
         /* Check any referrers first.  If we can invalidate them
            first, then we can invalidate this path as well. */

src/nix/flake.cc

@@ -4,6 +4,7 @@
 #include "shared.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "flake/flake.hh"
 #include "get-drvs.hh"
 #include "store-api.hh"

src/nix/main.cc

@@ -3,6 +3,7 @@
 #include "command.hh"
 #include "common-args.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "globals.hh"
 #include "legacy.hh"
 #include "shared.hh"

src/nix/repl.cc

@@ -1,4 +1,5 @@
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "globals.hh"
 #include "command.hh"
 #include "installable-value.hh"

src/nix/search.cc

@@ -2,6 +2,7 @@
 #include "globals.hh"
 #include "eval.hh"
 #include "eval-inline.hh"
+#include "eval-settings.hh"
 #include "names.hh"
 #include "get-drvs.hh"
 #include "common-args.hh"

src/nix/upgrade-nix.cc

@@ -3,6 +3,7 @@
 #include "store-api.hh"
 #include "filetransfer.hh"
 #include "eval.hh"
+#include "eval-settings.hh"
 #include "attr-path.hh"
 #include "names.hh"
 #include "progress-bar.hh"