Merge remote-tracking branch 'origin/master' into flakes

This commit is contained in:
Eelco Dolstra 2019-05-08 14:30:27 +02:00
commit 2bc55aba1e
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
23 changed files with 167 additions and 28 deletions

View file

@ -184,4 +184,7 @@ to be included. (This is the default.)</para>
the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link> the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link>
in your local <filename>nix.conf</filename>.</para> in your local <filename>nix.conf</filename>.</para>
<para>To build only on remote builders and disable building on the local machine,
you can use the option <option>--max-jobs 0</option>.</para>
</chapter> </chapter>

View file

@ -107,14 +107,22 @@
<varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option> <varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option>
<replaceable>number</replaceable></term> <replaceable>number</replaceable></term>
<listitem><para>Sets the maximum number of build jobs that Nix will <listitem>
<para>Sets the maximum number of build jobs that Nix will
perform in parallel to the specified number. Specify perform in parallel to the specified number. Specify
<literal>auto</literal> to use the number of CPUs in the system. <literal>auto</literal> to use the number of CPUs in the system.
The default is specified by the <link The default is specified by the <link
linkend='conf-max-jobs'><literal>max-jobs</literal></link> linkend='conf-max-jobs'><literal>max-jobs</literal></link>
configuration setting, which itself defaults to configuration setting, which itself defaults to
<literal>1</literal>. A higher value is useful on SMP systems or to <literal>1</literal>. A higher value is useful on SMP systems or to
exploit I/O latency.</para></listitem> exploit I/O latency.</para>
<para> Setting it to <literal>0</literal> disallows building on the local
machine, which is useful when you want builds to happen only on remote
builders.</para>
</listitem>
</varlistentry> </varlistentry>

View file

@ -705,6 +705,19 @@ builtins.genList (x: x * x) 5
</varlistentry> </varlistentry>
<varlistentry xml:id='builtin-hashFile'>
<term><function>builtins.hashFile</function>
<replaceable>type</replaceable> <replaceable>p</replaceable></term>
<listitem><para>Return a base-16 representation of the
cryptographic hash of the file at path <replaceable>p</replaceable>. The
hash algorithm specified by <replaceable>type</replaceable> must
be one of <literal>"md5"</literal>, <literal>"sha1"</literal>,
<literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem>
</varlistentry>
<varlistentry xml:id='builtin-head'> <varlistentry xml:id='builtin-head'>
<term><function>builtins.head</function> <term><function>builtins.head</function>
<replaceable>list</replaceable></term> <replaceable>list</replaceable></term>

View file

@ -24,11 +24,11 @@ symlinks to the files of the active applications. </para>
<para>Components are installed from a set of <emphasis>Nix <para>Components are installed from a set of <emphasis>Nix
expressions</emphasis> that tell Nix how to build those packages, expressions</emphasis> that tell Nix how to build those packages,
including, if necessary, their dependencies. There is a collection of including, if necessary, their dependencies. There is a collection of
Nix expressions called the Nix Package collection that contains Nix expressions called the Nixpkgs package collection that contains
packages ranging from basic development stuff such as GCC and Glibc, packages ranging from basic development stuff such as GCC and Glibc,
to end-user applications like Mozilla Firefox. (Nix is however not to end-user applications like Mozilla Firefox. (Nix is however not
tied to the Nix Package collection; you could write your own Nix tied to the Nixpkgs package collection; you could write your own Nix
expressions based on it, or completely new ones.)</para> expressions based on Nixpkgs, or completely new ones.)</para>
<para>You can manually download the latest version of Nixpkgs from <para>You can manually download the latest version of Nixpkgs from
<link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However, <link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However,

View file

@ -67,10 +67,10 @@ sub downloadFile {
} }
my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die;
my $sha256_actual = `nix hash-file --type sha256 '$dstFile'`; my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`;
chomp $sha256_actual; chomp $sha256_actual;
if ($sha256_expected ne $sha256_actual) { if ($sha256_expected ne $sha256_actual) {
print STDERR "file $dstFile is corrupt\n"; print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n";
exit 1; exit 1;
} }

View file

@ -240,10 +240,16 @@ EOF
} }
trap finish_fail EXIT trap finish_fail EXIT
channel_update_failed=0
function finish_success { function finish_success {
finish_cleanup finish_cleanup
ok "Alright! We're done!" ok "Alright! We're done!"
if [ "x$channel_update_failed" = x1 ]; then
echo ""
echo "But fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
cat <<EOF cat <<EOF
Before Nix will work in your existing shells, you'll need to close Before Nix will work in your existing shells, you'll need to close
@ -734,7 +740,9 @@ setup_default_profile() {
# otherwise it will be lost in environments where sudo doesn't pass # otherwise it will be lost in environments where sudo doesn't pass
# all the environment variables by default. # all the environment variables by default.
_sudo "to update the default channel in the default profile" \ _sudo "to update the default channel in the default profile" \
HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \
|| channel_update_failed=1
} }

View file

@ -22,10 +22,12 @@ if [ -z "$HOME" ]; then
exit 1 exit 1
fi fi
# macOS support for 10.10 or higher # macOS support for 10.12.6 or higher
if [ "$(uname -s)" = "Darwin" ]; then if [ "$(uname -s)" = "Darwin" ]; then
if [ $(($(sw_vers -productVersion | cut -d '.' -f 2))) -lt 10 ]; then macos_major=$(sw_vers -productVersion | cut -d '.' -f 2)
echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.10 or higher" macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3)
if [ "$macos_major" -lt 12 ] || ([ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]); then
echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher"
exit 1 exit 1
fi fi
fi fi
@ -132,7 +134,10 @@ if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then
$nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
fi fi
if [ -z "$_NIX_INSTALLER_TEST" ]; then if [ -z "$_NIX_INSTALLER_TEST" ]; then
$nix/bin/nix-channel --update nixpkgs if ! $nix/bin/nix-channel --update nixpkgs; then
echo "Fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"nix-channel --update nixpkgs\"."
fi
fi fi
added= added=

View file

@ -1820,6 +1820,7 @@ void EvalState::printStats()
gc.attr("totalBytes", totalBytes); gc.attr("totalBytes", totalBytes);
} }
#endif #endif
if (countCalls) { if (countCalls) {
{ {
auto obj = topObj.object("primops"); auto obj = topObj.object("primops");
@ -1855,6 +1856,11 @@ void EvalState::printStats()
} }
} }
} }
if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") {
auto list = topObj.list("symbols");
symbols.dump([&](const std::string & s) { list.elem(s); });
}
} }
} }

View file

@ -923,6 +923,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str());
} }
/* Return the cryptographic hash of a file in base-16. */
static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string type = state.forceStringNoCtx(*args[0], pos);
HashType ht = parseHashType(type);
if (ht == htUnknown)
throw Error(format("unknown hash type '%1%', at %2%") % type % pos);
PathSet context; // discarded
Path p = state.coerceToPath(pos, *args[1], context);
mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context);
}
/* Read a directory (without . or ..) */ /* Read a directory (without . or ..) */
static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v)
{ {
@ -2202,6 +2216,7 @@ void EvalState::createBaseEnv()
addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readFile", 1, prim_readFile);
addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__readDir", 1, prim_readDir);
addPrimOp("__findFile", 2, prim_findFile); addPrimOp("__findFile", 2, prim_findFile);
addPrimOp("__hashFile", 2, prim_hashFile);
// Creating files // Creating files
addPrimOp("__toXML", 1, prim_toXML); addPrimOp("__toXML", 1, prim_toXML);

View file

@ -75,6 +75,13 @@ public:
} }
size_t totalSize() const; size_t totalSize() const;
template<typename T>
void dump(T callback)
{
for (auto & s : symbols)
callback(s);
}
}; };
} }

View file

@ -803,6 +803,9 @@ private:
/* Whether we're currently doing a chroot build. */ /* Whether we're currently doing a chroot build. */
bool useChroot = false; bool useChroot = false;
/* Whether we need to perform hash rewriting if there are valid output paths. */
bool needsHashRewrite;
Path chrootRootDir; Path chrootRootDir;
/* RAII object to delete the chroot directory. */ /* RAII object to delete the chroot directory. */
@ -994,6 +997,13 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut
, wantedOutputs(wantedOutputs) , wantedOutputs(wantedOutputs)
, buildMode(buildMode) , buildMode(buildMode)
{ {
#if __linux__
needsHashRewrite = !useChroot;
#else
/* Darwin requires hash rewriting even when sandboxing is enabled. */
needsHashRewrite = true;
#endif
state = &DerivationGoal::getDerivation; state = &DerivationGoal::getDerivation;
name = (format("building of '%1%'") % drvPath).str(); name = (format("building of '%1%'") % drvPath).str();
trace("created"); trace("created");
@ -2073,7 +2083,7 @@ void DerivationGoal::startBuilder()
#endif #endif
} }
else { if (needsHashRewrite) {
if (pathExists(homeDir)) if (pathExists(homeDir))
throw Error(format("directory '%1%' exists; please remove it") % homeDir); throw Error(format("directory '%1%' exists; please remove it") % homeDir);
@ -2500,17 +2510,17 @@ void setupSeccomp()
seccomp_release(ctx); seccomp_release(ctx);
}); });
if (settings.thisSystem == "x86_64-linux" && if (nativeSystem == "x86_64-linux" &&
seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
throw SysError("unable to add 32-bit seccomp architecture"); throw SysError("unable to add 32-bit seccomp architecture");
if (settings.thisSystem == "x86_64-linux" && if (nativeSystem == "x86_64-linux" &&
seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
throw SysError("unable to add X32 seccomp architecture"); throw SysError("unable to add X32 seccomp architecture");
if (settings.thisSystem == "aarch64-linux" && if (nativeSystem == "aarch64-linux" &&
seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes."); printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
/* Prevent builders from creating setuid/setgid binaries. */ /* Prevent builders from creating setuid/setgid binaries. */
for (int perm : { S_ISUID, S_ISGID }) { for (int perm : { S_ISUID, S_ISGID }) {
@ -2873,6 +2883,10 @@ void DerivationGoal::runChild()
for (auto & i : missingPaths) { for (auto & i : missingPaths) {
sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str();
} }
/* Also add redirected outputs to the chroot */
for (auto & i : redirectedOutputs) {
sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str();
}
sandboxProfile += ")\n"; sandboxProfile += ")\n";
/* Our inputs (transitive dependencies and any impurities computed above) /* Our inputs (transitive dependencies and any impurities computed above)
@ -3051,7 +3065,9 @@ void DerivationGoal::registerOutputs()
throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path);
} }
if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path);
} else { }
if (needsHashRewrite) {
Path redirected = redirectedOutputs[path]; Path redirected = redirectedOutputs[path];
if (buildMode == bmRepair if (buildMode == bmRepair
&& redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && redirectedBadOutputs.find(path) != redirectedBadOutputs.end()

View file

@ -326,10 +326,9 @@ void LocalStore::findRootsNoTemp(Roots & roots, bool censor)
findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots);
findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots);
/* Add additional roots returned by the program specified by the /* Add additional roots returned by different platforms-specific
NIX_ROOT_FINDER environment variable. This is typically used heuristics. This is typically used to add running programs to
to add running programs to the set of roots (to prevent them the set of roots (to prevent them from being garbage collected). */
from being garbage collected). */
findRuntimeRoots(roots, censor); findRuntimeRoots(roots, censor);
} }

View file

@ -38,6 +38,9 @@ extern char * * environ;
namespace nix { namespace nix {
const std::string nativeSystem = SYSTEM;
BaseError & BaseError::addPrefix(const FormatOrString & fs) BaseError & BaseError::addPrefix(const FormatOrString & fs)
{ {
prefix_ = fs.s + prefix_; prefix_ = fs.s + prefix_;

View file

@ -30,6 +30,10 @@ struct Sink;
struct Source; struct Source;
/* The system for which Nix is compiled. */
extern const std::string nativeSystem;
/* Return an environment variable. */ /* Return an environment variable. */
string getEnv(const string & key, const string & def = ""); string getEnv(const string & key, const string & def = "");

View file

@ -274,13 +274,14 @@ static void _main(int argc, char * * argv)
exprs = {state->parseStdin()}; exprs = {state->parseStdin()};
else else
for (auto i : left) { for (auto i : left) {
if (fromArgs)
exprs.push_back(state->parseExprFromString(i, absPath(".")));
else {
auto absolute = i; auto absolute = i;
try { try {
absolute = canonPath(absPath(i), true); absolute = canonPath(absPath(i), true);
} catch (Error e) {}; } catch (Error e) {};
if (fromArgs) if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
exprs.push_back(state->parseExprFromString(i, absPath(".")));
else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?")))
drvs.push_back(DrvInfo(*state, store, absolute)); drvs.push_back(DrvInfo(*state, store, absolute));
else else
/* If we're in a #! script, interpret filenames /* If we're in a #! script, interpret filenames
@ -288,6 +289,7 @@ static void _main(int argc, char * * argv)
exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state,
inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))));
} }
}
/* Evaluate them into derivations. */ /* Evaluate them into derivations. */
if (attrPaths.empty()) attrPaths = {""}; if (attrPaths.empty()) attrPaths = {""};

View file

@ -192,6 +192,14 @@ static int listPossibleCallback(char *s, char ***avp) {
return ac; return ac;
} }
namespace {
// Used to communicate to NixRepl::getLine whether a signal occurred in ::readline.
volatile sig_atomic_t g_signal_received = 0;
void sigintHandler(int signo) {
g_signal_received = signo;
}
}
void NixRepl::mainLoop(const std::vector<std::string> & files) void NixRepl::mainLoop(const std::vector<std::string> & files)
{ {
@ -251,8 +259,40 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
bool NixRepl::getLine(string & input, const std::string &prompt) bool NixRepl::getLine(string & input, const std::string &prompt)
{ {
struct sigaction act, old;
sigset_t savedSignalMask, set;
auto setupSignals = [&]() {
act.sa_handler = sigintHandler;
sigfillset(&act.sa_mask);
act.sa_flags = 0;
if (sigaction(SIGINT, &act, &old))
throw SysError("installing handler for SIGINT");
sigemptyset(&set);
sigaddset(&set, SIGINT);
if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask))
throw SysError("unblocking SIGINT");
};
auto restoreSignals = [&]() {
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
if (sigaction(SIGINT, &old, 0))
throw SysError("restoring handler for SIGINT");
};
setupSignals();
char * s = readline(prompt.c_str()); char * s = readline(prompt.c_str());
Finally doFree([&]() { free(s); }); Finally doFree([&]() { free(s); });
restoreSignals();
if (g_signal_received) {
g_signal_received = 0;
input.clear();
return true;
}
if (!s) if (!s)
return false; return false;
input += s; input += s;

BIN
tests/lang/binary-data Normal file

Binary file not shown.

View file

@ -0,0 +1,5 @@
let
paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ];
in
toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]))

View file

@ -1 +0,0 @@
[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]

View file

@ -0,0 +1 @@
[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ]

View file

@ -0,0 +1,4 @@
let
paths = [ ./data ./binary-data ];
in
builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])

View file

@ -0,0 +1 @@
[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]