Merge remote-tracking branch 'upstream/master' into overlayfs-store

This commit is contained in:
John Ericson 2023-08-02 14:14:11 -04:00
commit 73c9fc7ab1
19 changed files with 224 additions and 186 deletions

View file

@ -21,4 +21,4 @@ jobs:
- uses: actions/labeler@v4 - uses: actions/labeler@v4
with: with:
repo-token: ${{ secrets.GITHUB_TOKEN }} repo-token: ${{ secrets.GITHUB_TOKEN }}
sync-labels: true sync-labels: false

View file

@ -1,3 +1,4 @@
#include "eval-settings.hh"
#include "common-eval-args.hh" #include "common-eval-args.hh"
#include "shared.hh" #include "shared.hh"
#include "filetransfer.hh" #include "filetransfer.hh"

View file

@ -11,6 +11,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "shared.hh" #include "shared.hh"

View file

@ -26,6 +26,7 @@ extern "C" {
#include "eval.hh" #include "eval.hh"
#include "eval-cache.hh" #include "eval-cache.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval-settings.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "store-api.hh" #include "store-api.hh"
#include "log-store.hh" #include "log-store.hh"

View file

@ -0,0 +1,95 @@
#include "globals.hh"
#include "profiles.hh"
#include "eval.hh"
#include "eval-settings.hh"
namespace nix {
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
static Strings parseNixPath(const std::string & s)
{
Strings res;
auto p = s.begin();
while (p != s.end()) {
auto start = p;
auto start2 = p;
while (p != s.end() && *p != ':') {
if (*p == '=') start2 = p + 1;
++p;
}
if (p == s.end()) {
if (p != start) res.push_back(std::string(start, p));
break;
}
if (*p == ':') {
auto prefix = std::string(start2, s.end());
if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
++p;
while (p != s.end() && *p != ':') ++p;
}
res.push_back(std::string(start, p));
if (p == s.end()) break;
}
++p;
}
return res;
}
EvalSettings::EvalSettings()
{
auto var = getEnv("NIX_PATH");
if (var) nixPath = parseNixPath(*var);
}
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathAccessible(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
}
return res;
}
bool EvalSettings::isPseudoUrl(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}
std::string EvalSettings::resolvePseudoUrl(std::string_view url)
{
if (hasPrefix(url, "channel:"))
return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
else
return std::string(url);
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
}

View file

@ -0,0 +1,98 @@
#pragma once
#include "config.hh"
namespace nix {
struct EvalSettings : Config
{
EvalSettings();
static Strings getDefaultNixPath();
static bool isPseudoUrl(std::string_view s);
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
R"(
List of directories to be searched for `<...>` file references
In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
)"};
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of the Nix search path (as set via the `NIX_PATH`
environment variable or the `-I` option), or to URIs outside of
[`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
The default is `false`.
)"};
Setting<bool> pureEval{this, false, "pure-eval",
R"(
Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
- Restrict file system and network access to files specified by cryptographic hash
- Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
)"
};
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
R"(
By default, Nix allows you to `import` from a derivation, allowing
building at evaluation time. With this option set to false, Nix will
throw an error when evaluating an expression that uses this feature,
allowing users to ensure their evaluation will not require any
builds to take place.
)"};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
R"(
A list of URI prefixes to which access is allowed in restricted
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
R"(
If set to `true`, the Nix evaluator will trace every function call.
Nix will print a log message at the "vomit" level for every function
entrance and function exit.
function-trace entered undefined position at 1565795816999559622
function-trace exited undefined position at 1565795816999581277
function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
The `undefined position` means the function call is a builtin.
Use the `contrib/stack-collapse.py` script distributed with the Nix
source code to convert the trace logs in to a format suitable for
`flamegraph.pl`.
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
R"(
If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
)"};
Setting<bool> traceVerbose{this, false, "trace-verbose",
"Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
};
extern EvalSettings evalSettings;
}

View file

@ -1,4 +1,5 @@
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "hash.hh" #include "hash.hh"
#include "types.hh" #include "types.hh"
#include "util.hh" #include "util.hh"
@ -420,44 +421,6 @@ void initGC()
} }
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
static Strings parseNixPath(const std::string & s)
{
Strings res;
auto p = s.begin();
while (p != s.end()) {
auto start = p;
auto start2 = p;
while (p != s.end() && *p != ':') {
if (*p == '=') start2 = p + 1;
++p;
}
if (p == s.end()) {
if (p != start) res.push_back(std::string(start, p));
break;
}
if (*p == ':') {
auto prefix = std::string(start2, s.end());
if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
++p;
while (p != s.end() && *p != ':') ++p;
}
res.push_back(std::string(start, p));
if (p == s.end()) break;
}
++p;
}
return res;
}
ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) ErrorBuilder & ErrorBuilder::atPos(PosIdx pos)
{ {
info.errPos = state.positions[pos]; info.errPos = state.positions[pos];
@ -2626,54 +2589,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
} }
EvalSettings::EvalSettings()
{
auto var = getEnv("NIX_PATH");
if (var) nixPath = parseNixPath(*var);
}
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathAccessible(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
}
return res;
}
bool EvalSettings::isPseudoUrl(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}
std::string EvalSettings::resolvePseudoUrl(std::string_view url)
{
if (hasPrefix(url, "channel:"))
return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
else
return std::string(url);
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
} }

View file

@ -787,98 +787,6 @@ struct InvalidPathError : EvalError
#endif #endif
}; };
struct EvalSettings : Config
{
EvalSettings();
static Strings getDefaultNixPath();
static bool isPseudoUrl(std::string_view s);
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
R"(
List of directories to be searched for `<...>` file references
In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
)"};
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of the Nix search path (as set via the `NIX_PATH`
environment variable or the `-I` option), or to URIs outside of
[`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
The default is `false`.
)"};
Setting<bool> pureEval{this, false, "pure-eval",
R"(
Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
- Restrict file system and network access to files specified by cryptographic hash
- Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
)"
};
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
R"(
By default, Nix allows you to `import` from a derivation, allowing
building at evaluation time. With this option set to false, Nix will
throw an error when evaluating an expression that uses this feature,
allowing users to ensure their evaluation will not require any
builds to take place.
)"};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
R"(
A list of URI prefixes to which access is allowed in restricted
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
R"(
If set to `true`, the Nix evaluator will trace every function call.
Nix will print a log message at the "vomit" level for every function
entrance and function exit.
function-trace entered undefined position at 1565795816999559622
function-trace exited undefined position at 1565795816999581277
function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
The `undefined position` means the function call is a builtin.
Use the `contrib/stack-collapse.py` script distributed with the Nix
source code to convert the trace logs in to a format suitable for
`flamegraph.pl`.
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
R"(
If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
)"};
Setting<bool> traceVerbose{this, false, "trace-verbose",
"Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
};
extern EvalSettings evalSettings;
static const std::string corepkgsPrefix{"/__corepkgs__/"}; static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType> template<class ErrorType>

View file

@ -1,5 +1,6 @@
#include "flake.hh" #include "flake.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "lockfile.hh" #include "lockfile.hh"
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"

View file

@ -22,6 +22,7 @@
#include "nixexpr.hh" #include "nixexpr.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh" #include "globals.hh"
namespace nix { namespace nix {

View file

@ -3,6 +3,7 @@
#include "downstream-placeholder.hh" #include "downstream-placeholder.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh" #include "globals.hh"
#include "json-to-value.hh" #include "json-to-value.hh"
#include "names.hh" #include "names.hh"

View file

@ -1,5 +1,6 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval-settings.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "url.hh" #include "url.hh"

View file

@ -1,5 +1,6 @@
#include "primops.hh" #include "primops.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval-settings.hh"
#include "store-api.hh" #include "store-api.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "filetransfer.hh" #include "filetransfer.hh"

View file

@ -1593,8 +1593,21 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths) bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPaths)
{ {
StringSet store; StorePathSet storePathsInStoreDir;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); /* Why aren't we using `queryAllValidPaths`? Because that would
tell us about all the paths than the database knows about. Here we
want to know about all the store paths in the store directory,
regardless of what the database thinks.
We will end up cross-referencing these two sources of truth (the
database and the filesystem) in the loop below, in order to catch
invalid states.
*/
for (auto & i : readDirectory(realStoreDir)) {
try {
storePathsInStoreDir.insert({i.name});
} catch (BadStorePath &) { }
}
/* Check whether all valid paths actually exist. */ /* Check whether all valid paths actually exist. */
printInfo("checking path existence..."); printInfo("checking path existence...");
@ -1603,7 +1616,7 @@ bool LocalStore::verifyAllValidPaths(RepairFlag repair, StorePathSet & validPath
bool errors = false; bool errors = false;
auto existsInStoreDir = [&](const StorePath & storePath) { auto existsInStoreDir = [&](const StorePath & storePath) {
return store.count(std::string(storePath.to_string())); return storePathsInStoreDir.count(storePath);
}; };
for (auto & i : queryAllValidPaths()) for (auto & i : queryAllValidPaths())
@ -1620,8 +1633,6 @@ void LocalStore::verifyPath(const StorePath & path, std::function<bool(const Sto
if (!done.insert(path).second) return; if (!done.insert(path).second) return;
auto pathS = printStorePath(path);
if (!existsInStoreDir(path)) { if (!existsInStoreDir(path)) {
/* Check any referrers first. If we can invalidate them /* Check any referrers first. If we can invalidate them
first, then we can invalidate this path as well. */ first, then we can invalidate this path as well. */

View file

@ -4,6 +4,7 @@
#include "shared.hh" #include "shared.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval-settings.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -3,6 +3,7 @@
#include "command.hh" #include "command.hh"
#include "common-args.hh" #include "common-args.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh" #include "globals.hh"
#include "legacy.hh" #include "legacy.hh"
#include "shared.hh" #include "shared.hh"

View file

@ -1,4 +1,5 @@
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh" #include "globals.hh"
#include "command.hh" #include "command.hh"
#include "installable-value.hh" #include "installable-value.hh"

View file

@ -2,6 +2,7 @@
#include "globals.hh" #include "globals.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-inline.hh" #include "eval-inline.hh"
#include "eval-settings.hh"
#include "names.hh" #include "names.hh"
#include "get-drvs.hh" #include "get-drvs.hh"
#include "common-args.hh" #include "common-args.hh"

View file

@ -3,6 +3,7 @@
#include "store-api.hh" #include "store-api.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "eval.hh" #include "eval.hh"
#include "eval-settings.hh"
#include "attr-path.hh" #include "attr-path.hh"
#include "names.hh" #include "names.hh"
#include "progress-bar.hh" #include "progress-bar.hh"