2018-09-24 14:53:44 +03:00
|
|
|
|
#include "daemon.hh"
|
|
|
|
|
#include "monitor-fd.hh"
|
|
|
|
|
#include "worker-protocol.hh"
|
|
|
|
|
#include "store-api.hh"
|
2021-03-02 05:50:41 +02:00
|
|
|
|
#include "path-with-outputs.hh"
|
2018-09-24 14:53:44 +03:00
|
|
|
|
#include "finally.hh"
|
|
|
|
|
#include "affinity.hh"
|
|
|
|
|
#include "archive.hh"
|
|
|
|
|
#include "derivations.hh"
|
|
|
|
|
#include "args.hh"
|
|
|
|
|
|
|
|
|
|
namespace nix::daemon {
|
|
|
|
|
|
|
|
|
|
Sink & operator << (Sink & sink, const Logger::Fields & fields)
|
|
|
|
|
{
|
|
|
|
|
sink << fields.size();
|
|
|
|
|
for (auto & f : fields) {
|
|
|
|
|
sink << f.type;
|
|
|
|
|
if (f.type == Logger::Field::tInt)
|
|
|
|
|
sink << f.i;
|
|
|
|
|
else if (f.type == Logger::Field::tString)
|
|
|
|
|
sink << f.s;
|
|
|
|
|
else abort();
|
|
|
|
|
}
|
|
|
|
|
return sink;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Logger that forwards log messages to the client, *if* we're in a
|
|
|
|
|
state where the protocol allows it (i.e., when canSendStderr is
|
|
|
|
|
true). */
|
|
|
|
|
struct TunnelLogger : public Logger
|
|
|
|
|
{
|
|
|
|
|
FdSink & to;
|
|
|
|
|
|
|
|
|
|
struct State
|
|
|
|
|
{
|
|
|
|
|
bool canSendStderr = false;
|
|
|
|
|
std::vector<std::string> pendingMsgs;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
Sync<State> state_;
|
|
|
|
|
|
|
|
|
|
unsigned int clientVersion;
|
|
|
|
|
|
|
|
|
|
TunnelLogger(FdSink & to, unsigned int clientVersion)
|
2020-07-02 18:04:31 +03:00
|
|
|
|
: to(to), clientVersion(clientVersion) { }
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
void enqueueMsg(const std::string & s)
|
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
|
|
|
|
|
if (state->canSendStderr) {
|
|
|
|
|
assert(state->pendingMsgs.empty());
|
|
|
|
|
try {
|
|
|
|
|
to(s);
|
|
|
|
|
to.flush();
|
|
|
|
|
} catch (...) {
|
|
|
|
|
/* Write failed; that means that the other side is
|
|
|
|
|
gone. */
|
|
|
|
|
state->canSendStderr = false;
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
} else
|
|
|
|
|
state->pendingMsgs.push_back(s);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void log(Verbosity lvl, const FormatOrString & fs) override
|
|
|
|
|
{
|
|
|
|
|
if (lvl > verbosity) return;
|
|
|
|
|
|
|
|
|
|
StringSink buf;
|
|
|
|
|
buf << STDERR_NEXT << (fs.s + "\n");
|
|
|
|
|
enqueueMsg(*buf.s);
|
|
|
|
|
}
|
|
|
|
|
|
2020-04-18 00:07:44 +03:00
|
|
|
|
void logEI(const ErrorInfo & ei) override
|
|
|
|
|
{
|
2020-04-20 02:16:51 +03:00
|
|
|
|
if (ei.level > verbosity) return;
|
|
|
|
|
|
2020-06-15 15:12:39 +03:00
|
|
|
|
std::stringstream oss;
|
2020-06-29 19:20:51 +03:00
|
|
|
|
showErrorInfo(oss, ei, false);
|
2020-04-20 02:16:51 +03:00
|
|
|
|
|
|
|
|
|
StringSink buf;
|
2020-07-08 16:53:14 +03:00
|
|
|
|
buf << STDERR_NEXT << oss.str();
|
2020-04-20 02:16:51 +03:00
|
|
|
|
enqueueMsg(*buf.s);
|
2020-04-18 00:07:44 +03:00
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
/* startWork() means that we're starting an operation for which we
|
2020-07-29 01:48:39 +03:00
|
|
|
|
want to send out stderr to the client. */
|
2018-09-24 14:53:44 +03:00
|
|
|
|
void startWork()
|
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
state->canSendStderr = true;
|
|
|
|
|
|
|
|
|
|
for (auto & msg : state->pendingMsgs)
|
|
|
|
|
to(msg);
|
|
|
|
|
|
|
|
|
|
state->pendingMsgs.clear();
|
|
|
|
|
|
|
|
|
|
to.flush();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* stopWork() means that we're done; stop sending stderr to the
|
|
|
|
|
client. */
|
2020-10-07 18:13:54 +03:00
|
|
|
|
void stopWork(const Error * ex = nullptr)
|
2018-09-24 14:53:44 +03:00
|
|
|
|
{
|
|
|
|
|
auto state(state_.lock());
|
|
|
|
|
|
|
|
|
|
state->canSendStderr = false;
|
|
|
|
|
|
2020-10-07 18:13:54 +03:00
|
|
|
|
if (!ex)
|
2018-09-24 14:53:44 +03:00
|
|
|
|
to << STDERR_LAST;
|
|
|
|
|
else {
|
2020-10-07 18:13:54 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 26) {
|
|
|
|
|
to << STDERR_ERROR << *ex;
|
|
|
|
|
} else {
|
|
|
|
|
to << STDERR_ERROR << ex->what() << ex->status;
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,
|
|
|
|
|
const std::string & s, const Fields & fields, ActivityId parent) override
|
|
|
|
|
{
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 20) {
|
|
|
|
|
if (!s.empty())
|
|
|
|
|
log(lvl, s + "...");
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
StringSink buf;
|
|
|
|
|
buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent;
|
|
|
|
|
enqueueMsg(*buf.s);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void stopActivity(ActivityId act) override
|
|
|
|
|
{
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
|
|
|
|
|
StringSink buf;
|
|
|
|
|
buf << STDERR_STOP_ACTIVITY << act;
|
|
|
|
|
enqueueMsg(*buf.s);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void result(ActivityId act, ResultType type, const Fields & fields) override
|
|
|
|
|
{
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 20) return;
|
|
|
|
|
StringSink buf;
|
|
|
|
|
buf << STDERR_RESULT << act << type << fields;
|
|
|
|
|
enqueueMsg(*buf.s);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct TunnelSink : Sink
|
|
|
|
|
{
|
|
|
|
|
Sink & to;
|
|
|
|
|
TunnelSink(Sink & to) : to(to) { }
|
2020-12-02 15:00:43 +02:00
|
|
|
|
void operator () (std::string_view data)
|
2018-09-24 14:53:44 +03:00
|
|
|
|
{
|
|
|
|
|
to << STDERR_WRITE;
|
2020-12-02 15:00:43 +02:00
|
|
|
|
writeString(data, to);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct TunnelSource : BufferedSource
|
|
|
|
|
{
|
|
|
|
|
Source & from;
|
|
|
|
|
BufferedSink & to;
|
|
|
|
|
TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
|
2020-12-02 15:10:56 +02:00
|
|
|
|
size_t readUnbuffered(char * data, size_t len) override
|
2018-09-24 14:53:44 +03:00
|
|
|
|
{
|
|
|
|
|
to << STDERR_READ << len;
|
|
|
|
|
to.flush();
|
|
|
|
|
size_t n = readString(data, len, from);
|
|
|
|
|
if (n == 0) throw EndOfFile("unexpected end-of-file");
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
struct ClientSettings
|
|
|
|
|
{
|
|
|
|
|
bool keepFailed;
|
|
|
|
|
bool keepGoing;
|
|
|
|
|
bool tryFallback;
|
|
|
|
|
Verbosity verbosity;
|
|
|
|
|
unsigned int maxBuildJobs;
|
|
|
|
|
time_t maxSilentTime;
|
|
|
|
|
bool verboseBuild;
|
|
|
|
|
unsigned int buildCores;
|
|
|
|
|
bool useSubstitutes;
|
|
|
|
|
StringMap overrides;
|
|
|
|
|
|
|
|
|
|
void apply(TrustedFlag trusted)
|
|
|
|
|
{
|
|
|
|
|
settings.keepFailed = keepFailed;
|
|
|
|
|
settings.keepGoing = keepGoing;
|
|
|
|
|
settings.tryFallback = tryFallback;
|
|
|
|
|
nix::verbosity = verbosity;
|
|
|
|
|
settings.maxBuildJobs.assign(maxBuildJobs);
|
|
|
|
|
settings.maxSilentTime = maxSilentTime;
|
|
|
|
|
settings.verboseBuild = verboseBuild;
|
|
|
|
|
settings.buildCores = buildCores;
|
|
|
|
|
settings.useSubstitutes = useSubstitutes;
|
|
|
|
|
|
|
|
|
|
for (auto & i : overrides) {
|
|
|
|
|
auto & name(i.first);
|
|
|
|
|
auto & value(i.second);
|
|
|
|
|
|
|
|
|
|
auto setSubstituters = [&](Setting<Strings> & res) {
|
|
|
|
|
if (name != res.name && res.aliases.count(name) == 0)
|
|
|
|
|
return false;
|
|
|
|
|
StringSet trusted = settings.trustedSubstituters;
|
|
|
|
|
for (auto & s : settings.substituters.get())
|
|
|
|
|
trusted.insert(s);
|
|
|
|
|
Strings subs;
|
|
|
|
|
auto ss = tokenizeString<Strings>(value);
|
|
|
|
|
for (auto & s : ss)
|
|
|
|
|
if (trusted.count(s))
|
|
|
|
|
subs.push_back(s);
|
2020-12-05 06:17:19 +02:00
|
|
|
|
else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
|
|
|
|
|
subs.push_back(s + "/");
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
else
|
|
|
|
|
warn("ignoring untrusted substituter '%s'", s);
|
|
|
|
|
res = subs;
|
|
|
|
|
return true;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
if (name == "ssh-auth-sock") // obsolete
|
|
|
|
|
;
|
2021-08-30 11:39:31 +03:00
|
|
|
|
else if (name == settings.experimentalFeatures.name) {
|
|
|
|
|
// We don’t want to forward the experimental features to
|
|
|
|
|
// the daemon, as that could cause some pretty weird stuff
|
2021-10-25 16:53:01 +03:00
|
|
|
|
if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
|
2021-08-30 11:39:31 +03:00
|
|
|
|
debug("Ignoring the client-specified experimental features");
|
|
|
|
|
}
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
else if (trusted
|
|
|
|
|
|| name == settings.buildTimeout.name
|
2021-10-27 14:09:13 +03:00
|
|
|
|
|| name == settings.buildRepeat.name
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
|| name == "connect-timeout"
|
|
|
|
|
|| (name == "builders" && value == ""))
|
|
|
|
|
settings.set(name, value);
|
|
|
|
|
else if (setSubstituters(settings.substituters))
|
|
|
|
|
;
|
|
|
|
|
else
|
2020-09-14 14:48:51 +03:00
|
|
|
|
debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
} catch (UsageError & e) {
|
|
|
|
|
warn(e.what());
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2021-04-05 16:48:18 +03:00
|
|
|
|
static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
|
2021-03-02 05:50:41 +02:00
|
|
|
|
{
|
2021-04-05 16:48:18 +03:00
|
|
|
|
std::vector<DerivedPath> reqs;
|
2021-05-03 02:12:23 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
|
2021-04-05 16:48:18 +03:00
|
|
|
|
reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
|
2021-03-02 05:50:41 +02:00
|
|
|
|
} else {
|
|
|
|
|
for (auto & s : readStrings<Strings>(from))
|
2021-04-05 16:48:18 +03:00
|
|
|
|
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
|
2021-03-02 05:50:41 +02:00
|
|
|
|
}
|
|
|
|
|
return reqs;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
static void performOp(TunnelLogger * logger, ref<Store> store,
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
|
2018-09-24 14:53:44 +03:00
|
|
|
|
Source & from, BufferedSink & to, unsigned int op)
|
|
|
|
|
{
|
|
|
|
|
switch (op) {
|
|
|
|
|
|
|
|
|
|
case wopIsValidPath: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
|
|
|
|
logger->startWork();
|
2018-09-24 14:53:44 +03:00
|
|
|
|
bool result = store->isValidPath(path);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << result;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryValidPaths: {
|
2020-09-30 03:41:18 +03:00
|
|
|
|
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2020-10-21 22:31:19 +03:00
|
|
|
|
|
|
|
|
|
SubstituteFlag substitute = NoSubstitute;
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
|
|
|
|
|
substitute = readInt(from) ? Substitute : NoSubstitute;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2020-10-21 22:31:19 +03:00
|
|
|
|
if (substitute) {
|
|
|
|
|
store->substitutePaths(paths);
|
|
|
|
|
}
|
|
|
|
|
auto res = store->queryValidPaths(paths, substitute);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, res);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopHasSubstitutes: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
StorePathSet paths; // FIXME
|
2020-06-16 23:20:18 +03:00
|
|
|
|
paths.insert(path);
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto res = store->querySubstitutablePaths(paths);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << (res.count(path) != 0);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQuerySubstitutablePaths: {
|
2020-09-30 03:41:18 +03:00
|
|
|
|
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto res = store->querySubstitutablePaths(paths);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, res);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryPathHash: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
auto hash = store->queryPathInfo(path)->narHash;
|
|
|
|
|
logger->stopWork();
|
2020-08-05 21:42:48 +03:00
|
|
|
|
to << hash.to_string(Base16, false);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryReferences:
|
|
|
|
|
case wopQueryReferrers:
|
|
|
|
|
case wopQueryValidDerivers:
|
|
|
|
|
case wopQueryDerivationOutputs: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
StorePathSet paths;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
if (op == wopQueryReferences)
|
2019-12-05 20:11:09 +02:00
|
|
|
|
for (auto & i : store->queryPathInfo(path)->references)
|
2020-06-16 23:20:18 +03:00
|
|
|
|
paths.insert(i);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
else if (op == wopQueryReferrers)
|
|
|
|
|
store->queryReferrers(path, paths);
|
|
|
|
|
else if (op == wopQueryValidDerivers)
|
|
|
|
|
paths = store->queryValidDerivers(path);
|
|
|
|
|
else paths = store->queryDerivationOutputs(path);
|
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, paths);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryDerivationOutputNames: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2020-06-12 14:04:52 +03:00
|
|
|
|
auto names = store->readDerivation(path).outputNames();
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << names;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2020-06-10 12:20:52 +03:00
|
|
|
|
case wopQueryDerivationOutputMap: {
|
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
|
|
|
|
logger->startWork();
|
2020-08-20 17:12:51 +03:00
|
|
|
|
auto outputs = store->queryPartialDerivationOutputMap(path);
|
2020-06-10 12:20:52 +03:00
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, outputs);
|
2020-06-10 12:20:52 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
case wopQueryDeriver: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto info = store->queryPathInfo(path);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << (info->deriver ? store->printStorePath(*info->deriver) : "");
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryPathFromHashPart: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto hashPart = readString(from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->queryPathFromHashPart(hashPart);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << (path ? store->printStorePath(*path) : "");
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopAddToStore: {
|
2020-09-17 20:27:11 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 25) {
|
|
|
|
|
auto name = readString(from);
|
|
|
|
|
auto camStr = readString(from);
|
2020-09-30 03:41:18 +03:00
|
|
|
|
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2020-09-18 11:06:34 +03:00
|
|
|
|
bool repairBool;
|
|
|
|
|
from >> repairBool;
|
|
|
|
|
auto repair = RepairFlag{repairBool};
|
2020-09-17 20:27:11 +03:00
|
|
|
|
|
|
|
|
|
logger->startWork();
|
2020-09-18 12:22:13 +03:00
|
|
|
|
auto pathInfo = [&]() {
|
2020-09-17 20:27:11 +03:00
|
|
|
|
// NB: FramedSource must be out of scope before logger->stopWork();
|
|
|
|
|
ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr);
|
|
|
|
|
FramedSource source(from);
|
2020-09-18 12:22:13 +03:00
|
|
|
|
// TODO this is essentially RemoteStore::addCAToStore. Move it up to Store.
|
2020-09-17 20:27:11 +03:00
|
|
|
|
return std::visit(overloaded {
|
2021-10-01 00:31:21 +03:00
|
|
|
|
[&](TextHashMethod &) {
|
2020-09-17 20:27:11 +03:00
|
|
|
|
// We could stream this by changing Store
|
|
|
|
|
std::string contents = source.drain();
|
2020-09-18 12:22:13 +03:00
|
|
|
|
auto path = store->addTextToStore(name, contents, refs, repair);
|
|
|
|
|
return store->queryPathInfo(path);
|
2020-09-17 20:27:11 +03:00
|
|
|
|
},
|
2021-10-01 00:31:21 +03:00
|
|
|
|
[&](FixedOutputHashMethod & fohm) {
|
2021-10-23 21:30:51 +03:00
|
|
|
|
auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs);
|
2020-09-18 12:22:13 +03:00
|
|
|
|
return store->queryPathInfo(path);
|
2020-09-17 20:27:11 +03:00
|
|
|
|
},
|
|
|
|
|
}, contentAddressMethod);
|
|
|
|
|
}();
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
|
2021-07-26 14:31:09 +03:00
|
|
|
|
pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion));
|
2020-09-17 20:27:11 +03:00
|
|
|
|
} else {
|
|
|
|
|
HashType hashAlgo;
|
|
|
|
|
std::string baseName;
|
|
|
|
|
FileIngestionMethod method;
|
|
|
|
|
{
|
|
|
|
|
bool fixed;
|
|
|
|
|
uint8_t recursive;
|
|
|
|
|
std::string hashAlgoRaw;
|
|
|
|
|
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
|
|
|
|
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
|
|
|
|
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
|
|
|
|
method = FileIngestionMethod { recursive };
|
|
|
|
|
/* Compatibility hack. */
|
|
|
|
|
if (!fixed) {
|
|
|
|
|
hashAlgoRaw = "sha256";
|
|
|
|
|
method = FileIngestionMethod::Recursive;
|
|
|
|
|
}
|
|
|
|
|
hashAlgo = parseHashType(hashAlgoRaw);
|
2020-03-29 08:04:55 +03:00
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
2021-11-30 22:53:10 +02:00
|
|
|
|
auto dumpSource = sinkToSource([&](Sink & saved) {
|
|
|
|
|
if (method == FileIngestionMethod::Recursive) {
|
2021-11-30 23:02:45 +02:00
|
|
|
|
/* We parse the NAR dump through into `saved` unmodified,
|
|
|
|
|
so why all this extra work? We still parse the NAR so
|
|
|
|
|
that we aren't sending arbitrary data to `saved`
|
|
|
|
|
unwittingly`, and we know when the NAR ends so we don't
|
|
|
|
|
consume the rest of `from` and can't parse another
|
|
|
|
|
command. (We don't trust `addToStoreFromDump` to not
|
|
|
|
|
eagerly consume the entire stream it's given, past the
|
|
|
|
|
length of the Nar. */
|
|
|
|
|
TeeSource savedNARSource(from, saved);
|
2021-11-30 22:53:10 +02:00
|
|
|
|
ParseSink sink; /* null sink; just parse the NAR */
|
|
|
|
|
parseDump(sink, savedNARSource);
|
2021-11-30 23:02:45 +02:00
|
|
|
|
} else {
|
|
|
|
|
/* Incrementally parse the NAR file, stripping the
|
|
|
|
|
metadata, and streaming the sole file we expect into
|
|
|
|
|
`saved`. */
|
|
|
|
|
RetrieveRegularNARSink savedRegular { saved };
|
2021-11-30 22:53:10 +02:00
|
|
|
|
parseDump(savedRegular, from);
|
2021-11-30 23:02:45 +02:00
|
|
|
|
if (!savedRegular.regular) throw Error("regular file expected");
|
|
|
|
|
}
|
2021-11-30 22:53:10 +02:00
|
|
|
|
});
|
2020-09-17 20:27:11 +03:00
|
|
|
|
logger->startWork();
|
2021-11-30 22:53:10 +02:00
|
|
|
|
auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo);
|
2020-09-17 20:27:11 +03:00
|
|
|
|
logger->stopWork();
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
2020-09-17 20:27:11 +03:00
|
|
|
|
to << store->printStorePath(path);
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2021-07-26 14:31:09 +03:00
|
|
|
|
case wopAddMultipleToStore: {
|
|
|
|
|
bool repair, dontCheckSigs;
|
|
|
|
|
from >> repair >> dontCheckSigs;
|
|
|
|
|
if (!trusted && dontCheckSigs)
|
|
|
|
|
dontCheckSigs = false;
|
|
|
|
|
|
|
|
|
|
logger->startWork();
|
|
|
|
|
FramedSource source(from);
|
|
|
|
|
store->addMultipleToStore(source,
|
|
|
|
|
RepairFlag{repair},
|
|
|
|
|
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
case wopAddTextToStore: {
|
|
|
|
|
string suffix = readString(from);
|
|
|
|
|
string s = readString(from);
|
2020-09-30 03:41:18 +03:00
|
|
|
|
auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->addTextToStore(suffix, s, refs, NoRepair);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << store->printStorePath(path);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopExportPath: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
readInt(from); // obsolete
|
|
|
|
|
logger->startWork();
|
|
|
|
|
TunnelSink sink(to);
|
|
|
|
|
store->exportPath(path, sink);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopImportPaths: {
|
|
|
|
|
logger->startWork();
|
|
|
|
|
TunnelSource source(from, to);
|
2020-07-13 18:37:44 +03:00
|
|
|
|
auto paths = store->importPaths(source,
|
2018-09-24 14:53:44 +03:00
|
|
|
|
trusted ? NoCheckSigs : CheckSigs);
|
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
Strings paths2;
|
|
|
|
|
for (auto & i : paths) paths2.push_back(store->printStorePath(i));
|
|
|
|
|
to << paths2;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopBuildPaths: {
|
2021-04-05 16:48:18 +03:00
|
|
|
|
auto drvs = readDerivedPaths(*store, clientVersion, from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
BuildMode mode = bmNormal;
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
|
|
|
|
|
mode = (BuildMode) readInt(from);
|
|
|
|
|
|
|
|
|
|
/* Repairing is not atomic, so disallowed for "untrusted"
|
|
|
|
|
clients. */
|
|
|
|
|
if (mode == bmRepair && !trusted)
|
|
|
|
|
throw Error("repairing is not allowed because you are not in 'trusted-users'");
|
|
|
|
|
}
|
|
|
|
|
logger->startWork();
|
|
|
|
|
store->buildPaths(drvs, mode);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopBuildDerivation: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto drvPath = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
BasicDerivation drv;
|
2020-07-12 18:26:30 +03:00
|
|
|
|
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
BuildMode buildMode = (BuildMode) readInt(from);
|
|
|
|
|
logger->startWork();
|
2020-08-11 16:43:29 +03:00
|
|
|
|
|
|
|
|
|
/* Content-addressed derivations are trustless because their output paths
|
|
|
|
|
are verified by their content alone, so any derivation is free to
|
|
|
|
|
try to produce such a path.
|
|
|
|
|
|
|
|
|
|
Input-addressed derivation output paths, however, are calculated
|
|
|
|
|
from the derivation closure that produced them---even knowing the
|
|
|
|
|
root derivation is not enough. That the output data actually came
|
|
|
|
|
from those derivations is fundamentally unverifiable, but the daemon
|
|
|
|
|
trusts itself on that matter. The question instead is whether the
|
|
|
|
|
submitted plan has rights to the output paths it wants to fill, and
|
|
|
|
|
at least the derivation closure proves that.
|
|
|
|
|
|
|
|
|
|
It would have been nice if input-address algorithm merely depended
|
|
|
|
|
on the build time closure, rather than depending on the derivation
|
|
|
|
|
closure. That would mean input-addressed paths used at build time
|
|
|
|
|
would just be trusted and not need their own evidence. This is in
|
|
|
|
|
fact fine as the same guarantees would hold *inductively*: either
|
|
|
|
|
the remote builder has those paths and already trusts them, or it
|
|
|
|
|
needs to build them too and thus their evidence must be provided in
|
|
|
|
|
turn. The advantage of this variant algorithm is that the evidence
|
|
|
|
|
for input-addressed paths which the remote builder already has
|
|
|
|
|
doesn't need to be sent again.
|
|
|
|
|
|
|
|
|
|
That said, now that we have floating CA derivations, it is better
|
|
|
|
|
that people just migrate to those which also solve this problem, and
|
|
|
|
|
others. It's the same migration difficulty with strictly more
|
|
|
|
|
benefit.
|
|
|
|
|
|
|
|
|
|
Lastly, do note that when we parse fixed-output content-addressed
|
|
|
|
|
derivations, we throw out the precomputed output paths and just
|
|
|
|
|
store the hashes, so there aren't two competing sources of truth an
|
|
|
|
|
attacker could exploit. */
|
|
|
|
|
if (drv.type() == DerivationType::InputAddressed && !trusted)
|
|
|
|
|
throw Error("you are not privileged to build input-addressed derivations");
|
|
|
|
|
|
|
|
|
|
/* Make sure that the non-input-addressed derivations that got this far
|
|
|
|
|
are in fact content-addressed if we don't trust them. */
|
|
|
|
|
assert(derivationIsCA(drv.type()) || trusted);
|
|
|
|
|
|
2020-08-22 23:44:47 +03:00
|
|
|
|
/* Recompute the derivation path when we cannot trust the original. */
|
|
|
|
|
if (!trusted) {
|
|
|
|
|
/* Recomputing the derivation path for input-address derivations
|
|
|
|
|
makes it harder to audit them after the fact, since we need the
|
|
|
|
|
original not-necessarily-resolved derivation to verify the drv
|
|
|
|
|
derivation as adequate claim to the input-addressed output
|
|
|
|
|
paths. */
|
|
|
|
|
assert(derivationIsCA(drv.type()));
|
|
|
|
|
|
|
|
|
|
Derivation drv2;
|
|
|
|
|
static_cast<BasicDerivation &>(drv2) = drv;
|
|
|
|
|
drvPath = writeDerivation(*store, Derivation { drv2 });
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
auto res = store->buildDerivation(drvPath, drv, buildMode);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << res.status << res.errorMsg;
|
2021-02-28 20:42:46 +02:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
|
2021-03-22 17:18:48 +02:00
|
|
|
|
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
|
2021-02-28 20:42:46 +02:00
|
|
|
|
}
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
|
2021-01-25 12:08:38 +02:00
|
|
|
|
worker_proto::write(*store, to, res.builtOutputs);
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopEnsurePath: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
store->ensurePath(path);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopAddTempRoot: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
store->addTempRoot(path);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopAddIndirectRoot: {
|
|
|
|
|
Path path = absPath(readString(from));
|
|
|
|
|
logger->startWork();
|
|
|
|
|
store->addIndirectRoot(path);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2021-08-17 19:53:14 +03:00
|
|
|
|
// Obsolete.
|
2018-09-24 14:53:44 +03:00
|
|
|
|
case wopSyncWithGC: {
|
|
|
|
|
logger->startWork();
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopFindRoots: {
|
|
|
|
|
logger->startWork();
|
|
|
|
|
Roots roots = store->findRoots(!trusted);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
|
|
|
|
|
size_t size = 0;
|
|
|
|
|
for (auto & i : roots)
|
|
|
|
|
size += i.second.size();
|
|
|
|
|
|
|
|
|
|
to << size;
|
|
|
|
|
|
|
|
|
|
for (auto & [target, links] : roots)
|
|
|
|
|
for (auto & link : links)
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << link << store->printStorePath(target);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopCollectGarbage: {
|
|
|
|
|
GCOptions options;
|
|
|
|
|
options.action = (GCOptions::GCAction) readInt(from);
|
2020-09-30 03:41:18 +03:00
|
|
|
|
options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2018-09-24 14:53:44 +03:00
|
|
|
|
from >> options.ignoreLiveness >> options.maxFreed;
|
|
|
|
|
// obsolete fields
|
|
|
|
|
readInt(from);
|
|
|
|
|
readInt(from);
|
|
|
|
|
readInt(from);
|
|
|
|
|
|
|
|
|
|
GCResults results;
|
|
|
|
|
|
|
|
|
|
logger->startWork();
|
|
|
|
|
if (options.ignoreLiveness)
|
|
|
|
|
throw Error("you are not allowed to ignore liveness");
|
|
|
|
|
store->collectGarbage(options, results);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
|
|
|
|
|
to << results.paths << results.bytesFreed << 0 /* obsolete */;
|
|
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopSetOptions: {
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
|
|
|
|
|
ClientSettings clientSettings;
|
|
|
|
|
|
|
|
|
|
clientSettings.keepFailed = readInt(from);
|
|
|
|
|
clientSettings.keepGoing = readInt(from);
|
|
|
|
|
clientSettings.tryFallback = readInt(from);
|
|
|
|
|
clientSettings.verbosity = (Verbosity) readInt(from);
|
|
|
|
|
clientSettings.maxBuildJobs = readInt(from);
|
|
|
|
|
clientSettings.maxSilentTime = readInt(from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
readInt(from); // obsolete useBuildHook
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
readInt(from); // obsolete logType
|
|
|
|
|
readInt(from); // obsolete printBuildTrace
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
clientSettings.buildCores = readInt(from);
|
|
|
|
|
clientSettings.useSubstitutes = readInt(from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 12) {
|
|
|
|
|
unsigned int n = readInt(from);
|
|
|
|
|
for (unsigned int i = 0; i < n; i++) {
|
|
|
|
|
string name = readString(from);
|
|
|
|
|
string value = readString(from);
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
clientSettings.overrides.emplace(name, value);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
logger->startWork();
|
|
|
|
|
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
// FIXME: use some setting in recursive mode. Will need to use
|
|
|
|
|
// non-global variables.
|
|
|
|
|
if (!recursive)
|
|
|
|
|
clientSettings.apply(trusted);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQuerySubstitutablePathInfo: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
SubstitutablePathInfos infos;
|
2020-06-17 22:03:05 +03:00
|
|
|
|
store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto i = infos.find(path);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
if (i == infos.end())
|
|
|
|
|
to << 0;
|
|
|
|
|
else {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << 1
|
|
|
|
|
<< (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, i->second.references);
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << i->second.downloadSize
|
|
|
|
|
<< i->second.narSize;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQuerySubstitutablePathInfos: {
|
|
|
|
|
SubstitutablePathInfos infos;
|
2020-06-17 22:03:05 +03:00
|
|
|
|
StorePathCAMap pathsMap = {};
|
2020-06-20 01:06:19 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
|
2020-09-30 03:41:18 +03:00
|
|
|
|
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2020-06-20 01:06:19 +03:00
|
|
|
|
for (auto & path : paths)
|
|
|
|
|
pathsMap.emplace(path, std::nullopt);
|
|
|
|
|
} else
|
2020-09-30 03:41:18 +03:00
|
|
|
|
pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {});
|
2020-06-20 01:06:19 +03:00
|
|
|
|
logger->startWork();
|
2020-06-17 22:03:05 +03:00
|
|
|
|
store->querySubstitutablePathInfos(pathsMap, infos);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << infos.size();
|
|
|
|
|
for (auto & i : infos) {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << store->printStorePath(i.first)
|
|
|
|
|
<< (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, i.second.references);
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << i.second.downloadSize << i.second.narSize;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryAllValidPaths: {
|
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto paths = store->queryAllValidPaths();
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, paths);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryPathInfo: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
std::shared_ptr<const ValidPathInfo> info;
|
|
|
|
|
logger->startWork();
|
|
|
|
|
try {
|
|
|
|
|
info = store->queryPathInfo(path);
|
|
|
|
|
} catch (InvalidPath &) {
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw;
|
|
|
|
|
}
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
if (info) {
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
|
|
|
|
|
to << 1;
|
2021-07-26 14:31:09 +03:00
|
|
|
|
info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
} else {
|
|
|
|
|
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
|
|
|
|
|
to << 0;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopOptimiseStore:
|
|
|
|
|
logger->startWork();
|
|
|
|
|
store->optimiseStore();
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case wopVerifyStore: {
|
|
|
|
|
bool checkContents, repair;
|
|
|
|
|
from >> checkContents >> repair;
|
|
|
|
|
logger->startWork();
|
|
|
|
|
if (repair && !trusted)
|
|
|
|
|
throw Error("you are not privileged to repair paths");
|
|
|
|
|
bool errors = store->verifyStore(checkContents, (RepairFlag) repair);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << errors;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopAddSignatures: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
StringSet sigs = readStrings<StringSet>(from);
|
|
|
|
|
logger->startWork();
|
|
|
|
|
if (!trusted)
|
|
|
|
|
throw Error("you are not privileged to add signatures");
|
|
|
|
|
store->addSignatures(path, sigs);
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
to << 1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopNarFromPath: {
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
logger->stopWork();
|
2020-08-15 00:59:31 +03:00
|
|
|
|
dumpPath(store->toRealPath(path), to);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopAddToStoreNar: {
|
|
|
|
|
bool repair, dontCheckSigs;
|
2020-08-06 21:31:48 +03:00
|
|
|
|
auto path = store->parseStorePath(readString(from));
|
2019-12-05 20:11:09 +02:00
|
|
|
|
auto deriver = readString(from);
|
2020-08-06 21:31:48 +03:00
|
|
|
|
auto narHash = Hash::parseAny(readString(from), htSHA256);
|
|
|
|
|
ValidPathInfo info { path, narHash };
|
2019-12-05 20:11:09 +02:00
|
|
|
|
if (deriver != "")
|
|
|
|
|
info.deriver = store->parseStorePath(deriver);
|
2020-09-30 03:41:18 +03:00
|
|
|
|
info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
|
2018-09-24 14:53:44 +03:00
|
|
|
|
from >> info.registrationTime >> info.narSize >> info.ultimate;
|
|
|
|
|
info.sigs = readStrings<StringSet>(from);
|
2020-06-04 23:54:55 +03:00
|
|
|
|
info.ca = parseContentAddressOpt(readString(from));
|
2020-06-02 03:37:43 +03:00
|
|
|
|
from >> repair >> dontCheckSigs;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
if (!trusted && dontCheckSigs)
|
|
|
|
|
dontCheckSigs = false;
|
|
|
|
|
if (!trusted)
|
|
|
|
|
info.ultimate = false;
|
|
|
|
|
|
2020-07-29 01:48:39 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
|
|
|
|
|
logger->startWork();
|
|
|
|
|
{
|
|
|
|
|
FramedSource source(from);
|
|
|
|
|
store->addToStore(info, source, (RepairFlag) repair,
|
|
|
|
|
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
|
|
|
|
}
|
|
|
|
|
logger->stopWork();
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
|
2020-07-29 01:48:39 +03:00
|
|
|
|
else {
|
|
|
|
|
std::unique_ptr<Source> source;
|
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
|
|
|
|
|
source = std::make_unique<TunnelSource>(from, to);
|
|
|
|
|
else {
|
|
|
|
|
StringSink saved;
|
|
|
|
|
TeeSource tee { from, saved };
|
|
|
|
|
ParseSink ether;
|
|
|
|
|
parseDump(ether, tee);
|
|
|
|
|
source = std::make_unique<StringSource>(std::move(*saved.s));
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
2020-07-29 01:48:39 +03:00
|
|
|
|
logger->startWork();
|
|
|
|
|
|
|
|
|
|
// FIXME: race if addToStore doesn't read source?
|
|
|
|
|
store->addToStore(info, *source, (RepairFlag) repair,
|
|
|
|
|
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
|
|
|
|
|
|
|
|
|
logger->stopWork();
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryMissing: {
|
2021-04-05 16:48:18 +03:00
|
|
|
|
auto targets = readDerivedPaths(*store, clientVersion, from);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
logger->startWork();
|
2019-12-05 20:11:09 +02:00
|
|
|
|
StorePathSet willBuild, willSubstitute, unknown;
|
2020-07-30 14:10:49 +03:00
|
|
|
|
uint64_t downloadSize, narSize;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
|
|
|
|
logger->stopWork();
|
2020-09-30 03:41:18 +03:00
|
|
|
|
worker_proto::write(*store, to, willBuild);
|
|
|
|
|
worker_proto::write(*store, to, willSubstitute);
|
|
|
|
|
worker_proto::write(*store, to, unknown);
|
2019-12-05 20:11:09 +02:00
|
|
|
|
to << downloadSize << narSize;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-08 18:36:51 +03:00
|
|
|
|
case wopRegisterDrvOutput: {
|
|
|
|
|
logger->startWork();
|
2021-05-07 12:06:17 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
|
|
|
|
|
auto outputId = DrvOutput::parse(readString(from));
|
|
|
|
|
auto outputPath = StorePath(readString(from));
|
|
|
|
|
store->registerDrvOutput(Realisation{
|
|
|
|
|
.id = outputId, .outPath = outputPath});
|
|
|
|
|
} else {
|
|
|
|
|
auto realisation = worker_proto::read(*store, from, Phantom<Realisation>());
|
|
|
|
|
store->registerDrvOutput(realisation);
|
|
|
|
|
}
|
2020-10-08 18:36:51 +03:00
|
|
|
|
logger->stopWork();
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
case wopQueryRealisation: {
|
|
|
|
|
logger->startWork();
|
|
|
|
|
auto outputId = DrvOutput::parse(readString(from));
|
|
|
|
|
auto info = store->queryRealisation(outputId);
|
|
|
|
|
logger->stopWork();
|
2021-05-07 12:06:17 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) < 31) {
|
|
|
|
|
std::set<StorePath> outPaths;
|
|
|
|
|
if (info) outPaths.insert(info->outPath);
|
|
|
|
|
worker_proto::write(*store, to, outPaths);
|
|
|
|
|
} else {
|
|
|
|
|
std::set<Realisation> realisations;
|
|
|
|
|
if (info) realisations.insert(*info);
|
|
|
|
|
worker_proto::write(*store, to, realisations);
|
|
|
|
|
}
|
2020-10-08 18:36:51 +03:00
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 14:53:44 +03:00
|
|
|
|
default:
|
2020-04-22 02:07:07 +03:00
|
|
|
|
throw Error("invalid operation %1%", op);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void processConnection(
|
2018-09-25 13:49:20 +03:00
|
|
|
|
ref<Store> store,
|
2018-09-24 14:53:44 +03:00
|
|
|
|
FdSource & from,
|
|
|
|
|
FdSink & to,
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
TrustedFlag trusted,
|
|
|
|
|
RecursiveFlag recursive,
|
2020-08-12 18:14:56 +03:00
|
|
|
|
std::function<void(Store &)> authHook)
|
2018-09-24 14:53:44 +03:00
|
|
|
|
{
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
/* Exchange the greeting. */
|
|
|
|
|
unsigned int magic = readInt(from);
|
|
|
|
|
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
|
|
|
|
|
to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
|
|
|
|
|
to.flush();
|
|
|
|
|
unsigned int clientVersion = readInt(from);
|
|
|
|
|
|
|
|
|
|
if (clientVersion < 0x10a)
|
|
|
|
|
throw Error("the Nix client version is too old");
|
|
|
|
|
|
|
|
|
|
auto tunnelLogger = new TunnelLogger(to, clientVersion);
|
|
|
|
|
auto prevLogger = nix::logger;
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
// FIXME
|
|
|
|
|
if (!recursive)
|
|
|
|
|
logger = tunnelLogger;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
unsigned int opCount = 0;
|
|
|
|
|
|
|
|
|
|
Finally finally([&]() {
|
|
|
|
|
_isInterrupted = false;
|
|
|
|
|
prevLogger->log(lvlDebug, fmt("%d operations", opCount));
|
|
|
|
|
});
|
|
|
|
|
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) {
|
|
|
|
|
auto affinity = readInt(from);
|
|
|
|
|
setAffinityTo(affinity);
|
|
|
|
|
}
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
readInt(from); // obsolete reserveSpace
|
|
|
|
|
|
|
|
|
|
/* Send startup error messages to the client. */
|
|
|
|
|
tunnelLogger->startWork();
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
|
|
|
|
|
/* If we can't accept clientVersion, then throw an error
|
|
|
|
|
*here* (not above). */
|
2020-08-12 18:14:56 +03:00
|
|
|
|
authHook(*store);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
|
|
|
|
|
tunnelLogger->stopWork();
|
|
|
|
|
to.flush();
|
|
|
|
|
|
|
|
|
|
/* Process client requests. */
|
|
|
|
|
while (true) {
|
|
|
|
|
WorkerOp op;
|
|
|
|
|
try {
|
|
|
|
|
op = (WorkerOp) readInt(from);
|
|
|
|
|
} catch (Interrupted & e) {
|
|
|
|
|
break;
|
|
|
|
|
} catch (EndOfFile & e) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
opCount++;
|
|
|
|
|
|
|
|
|
|
try {
|
Recursive Nix support
This allows Nix builders to call Nix to build derivations, with some
limitations.
Example:
let nixpkgs = fetchTarball channel:nixos-18.03; in
with import <nixpkgs> {};
runCommand "foo"
{
buildInputs = [ nix jq ];
NIX_PATH = "nixpkgs=${nixpkgs}";
}
''
hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "hello-3.5"; })')
$hello/bin/hello
mkdir -p $out/bin
ln -s $hello/bin/hello $out/bin/hello
nix path-info -r --json $hello | jq .
''
This derivation makes a recursive Nix call to build GNU Hello and
symlinks it from its $out, i.e.
# ll ./result/bin/
lrwxrwxrwx 1 root root 63 Jan 1 1970 hello -> /nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5/bin/hello
# nix-store -qR ./result
/nix/store/hwwqshlmazzjzj7yhrkyjydxamvvkfd3-glibc-2.26-131
/nix/store/s0awxrs71gickhaqdwxl506hzccb30y5-hello-3.5
/nix/store/sgmvvyw8vhfqdqb619bxkcpfn9lvd8ss-foo
This is implemented as follows:
* Before running the outer builder, Nix creates a Unix domain socket
'.nix-socket' in the builder's temporary directory and sets
$NIX_REMOTE to point to it. It starts a thread to process
connections to this socket. (Thus you don't need to have nix-daemon
running.)
* The daemon thread uses a wrapper store (RestrictedStore) to keep
track of paths added through recursive Nix calls, to implement some
restrictions (see below), and to do some censorship (e.g. for
purity, queryPathInfo() won't return impure information such as
signatures and timestamps).
* After the build finishes, the output paths are scanned for
references to the paths added through recursive Nix calls (in
addition to the inputs closure). Thus, in the example above, $out
has a reference to $hello.
The main restriction on recursive Nix calls is that they cannot do
arbitrary substitutions. For example, doing
nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10
is forbidden unless /nix/store/kmwd... is in the inputs closure or
previously built by a recursive Nix call. This is to prevent
irreproducible derivations that have hidden dependencies on
substituters or the current store contents. Building a derivation is
fine, however, and Nix will use substitutes if available. In other
words, the builder has to present proof that it knows how to build a
desired store path from scratch by constructing a derivation graph for
that path.
Probably we should also disallow instantiating/building fixed-output
derivations (specifically, those that access the network, but
currently we have no way to mark fixed-output derivations that don't
access the network). Otherwise sandboxed derivations can bypass
sandbox restrictions and access the network.
When sandboxing is enabled, we make paths appear in the sandbox of the
builder by entering the mount namespace of the builder and
bind-mounting each path. This is tricky because we do a pivot_root()
in the builder to change the root directory of its mount namespace,
and thus the host /nix/store is not visible in the mount namespace of
the builder. To get around this, just before doing pivot_root(), we
branch a second mount namespace that shares its /nix/store mountpoint
with the parent.
Recursive Nix currently doesn't work on macOS in sandboxed mode
(because we can't change the sandbox policy of a running build) and in
non-root mode (because setns() barfs).
2018-10-02 17:01:26 +03:00
|
|
|
|
performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
} catch (Error & e) {
|
|
|
|
|
/* If we're not in a state where we can send replies, then
|
|
|
|
|
something went wrong processing the input of the
|
|
|
|
|
client. This can happen especially if I/O errors occur
|
|
|
|
|
during addTextToStore() / importPath(). If that
|
|
|
|
|
happens, just send the error message and exit. */
|
|
|
|
|
bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr;
|
2020-10-07 18:13:54 +03:00
|
|
|
|
tunnelLogger->stopWork(&e);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
if (!errorAllowed) throw;
|
|
|
|
|
} catch (std::bad_alloc & e) {
|
2020-10-07 18:13:54 +03:00
|
|
|
|
auto ex = Error("Nix daemon out of memory");
|
|
|
|
|
tunnelLogger->stopWork(&ex);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
to.flush();
|
|
|
|
|
|
|
|
|
|
assert(!tunnelLogger->state_.lock()->canSendStderr);
|
|
|
|
|
};
|
|
|
|
|
|
2020-10-07 18:13:54 +03:00
|
|
|
|
} catch (Error & e) {
|
|
|
|
|
tunnelLogger->stopWork(&e);
|
|
|
|
|
to.flush();
|
|
|
|
|
return;
|
2018-09-24 14:53:44 +03:00
|
|
|
|
} catch (std::exception & e) {
|
2020-10-07 18:13:54 +03:00
|
|
|
|
auto ex = Error(e.what());
|
|
|
|
|
tunnelLogger->stopWork(&ex);
|
2018-09-24 14:53:44 +03:00
|
|
|
|
to.flush();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|