diff --git a/configure.ac b/configure.ac index e587bd563..bb3f92e4d 100644 --- a/configure.ac +++ b/configure.ac @@ -184,7 +184,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a23b87e4e..7a012250d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,10 +203,9 @@ Most Nix commands accept the following command-line options: instead. - [`-I`](#opt-I) *path*\ - Add a path to the Nix expression search path. This option may be - given multiple times. See the `NIX_PATH` environment variable for - information on the semantics of the Nix search path. Paths added - through `-I` take precedence over `NIX_PATH`. + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a9782be5c..eeb19ad50 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -127,7 +127,7 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. - - Nix database{#gloss-nix-database}\ + - [Nix database]{#gloss-nix-database}\ An SQlite database to track [reference]s between [store object]s. This is an implementation detail of the [local store]. diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index e3fd962bd..525654d35 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -136,7 +136,7 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing +1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash @@ -153,6 +153,7 @@ which you may remove. ```console sudo mv /etc/zshrc.backup-before-nix /etc/zshrc sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc ``` This will stop shells from sourcing the file and bringing everything you diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de91dc28d..41ecbbeb4 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,8 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - loadConfFile(); - settings.lockCPU = false; _store = openStore(); } catch (Error & e) { croak("%s", e.what()); @@ -295,7 +293,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index cfc4baaca..ce9c7f45a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,8 +311,9 @@ connected: auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); - assert(result.builtOutputs.count(thisOutputId)); - auto newRealisation = result.builtOutputs.at(thisOutputId); + auto i = result.builtOutputs.find(outputName); + assert(i != result.builtOutputs.end()); + auto & newRealisation = i->second; missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 19e982df1..f0d322e6d 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -106,10 +106,10 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } else if (v.type() == nString) { - PathSet context; + NixStringContext context; auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); auto storePath = state->store->maybeParseStorePath(s); - if (storePath && context.count(std::string(s))) { + if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) { return {{ .path = DerivedPath::Opaque { .path = std::move(*storePath), diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1e0e2c9d5..a2b882355 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -593,8 +593,8 @@ std::vector, BuiltPathWithResult>> Installable::build std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); + for (auto & [outputName, realisation] : buildResult.builtOutputs) + outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { .path = BuiltPath::Built { bfd.drvPath, outputs }, .info = aux.info, diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index f41a0dadc..4b160a100 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -593,7 +593,7 @@ bool NixRepl::processLine(std::string line) const auto [path, line] = [&] () -> std::pair { if (v.type() == nPath || v.type() == nString) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { @@ -936,7 +936,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isDrv) { str << "«derivation "; Bindings::iterator i = v.attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i != v.attrs->end()) str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); else diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 8ae4270e6..ab654c1b0 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -118,7 +118,7 @@ std::pair findPackageFilename(EvalState & state, Value & v // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - PathSet context; + NixStringContext context; auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation"); auto fn = path.path.abs(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index ec4ad2f5e..ac646af8a 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(NixStringContextElem::parse(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -621,9 +621,11 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; - else if (v.type() == nPath) + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.string.s, std::move(context)}; + } else if (v.type() == nPath) return {v.path().to_string(), {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d6daf2727..e2b455b91 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -610,8 +610,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } SourcePath EvalState::checkSourcePath(const SourcePath & path_) @@ -693,7 +692,7 @@ void EvalState::checkURI(const std::string & uri) } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -945,25 +944,25 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const PathSet & context) +static void copyContextToValue(Value & v, const NixStringContext & context) { if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); + v.string.context[n++] = dupString(i.to_string().c_str()); v.string.context[n] = 0; } } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); @@ -1039,6 +1038,16 @@ void EvalState::mkPos(Value & v, PosIdx p) } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -1901,7 +1910,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector s; size_t sSize = 0; NixInt n = 0; @@ -2110,26 +2119,15 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { if (v.string.context) for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); + context.insert(NixStringContextElem::parse(*p)); } -NixStringContext Value::getContext(const Store & store) -{ - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(NixStringContextElem::parse(store, *p)); - return res; -} - - -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); copyContext(v, context); @@ -2159,7 +2157,7 @@ bool EvalState::isDerivation(Value & v) std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) + NixStringContext & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { @@ -2176,7 +2174,7 @@ std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & BackedStringView EvalState::coerceToString( const PosIdx pos, Value & v, - PathSet & context, + NixStringContext & context, std::string_view errorCtx, bool coerceMore, bool copyToStore, @@ -2258,7 +2256,7 @@ BackedStringView EvalState::coerceToString( } -StorePath EvalState::copyPathToStore(PathSet & context, const SourcePath & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); @@ -2275,12 +2273,14 @@ StorePath EvalState::copyPathToStore(PathSet & context, const SourcePath & path) return dstPath; }(); - context.insert(store->printStorePath(dstPath)); + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') @@ -2289,7 +2289,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & contex } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) @@ -2496,7 +2496,7 @@ void EvalState::printStats() } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string", showType()) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index dea8b4a3a..bb3ac2b22 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -57,7 +57,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); -void copyContext(const Value & v, PathSet & context); +void copyContext(const Value & v, NixStringContext & context); std::string printValue(const EvalState & state, const Value & v); @@ -330,7 +330,7 @@ public: * intended to distinguish between import-from-derivation and * sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + Path toRealPath(const Path & path, const NixStringContext & context); /** * Parse a Nix expression from the specified file. @@ -426,7 +426,7 @@ public: */ void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); [[gnu::noinline]] @@ -442,7 +442,7 @@ public: bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); /** * String coercion. @@ -452,12 +452,12 @@ public: * booleans and lists to a string. If `copyToStore` is set, * referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(PathSet & context, const SourcePath & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); /** * Path coercion. @@ -466,12 +466,12 @@ public: * path. The result is guaranteed to be a canonicalised, absolute * path. Nothing is copied to the store. */ - SourcePath coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** * Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); public: @@ -576,6 +576,12 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); + /* Create a string representing a store path. + + The string is the printed store path with a context containing a single + `Opaque` element of that store path. */ + void mkStorePathString(const StorePath & storePath, Value & v); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** @@ -587,7 +593,7 @@ public: * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context); private: diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 89ddbde7e..e89014862 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ccf868361..60bb6a71e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -265,7 +265,7 @@ static Flake getFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; + NixStringContext emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1602fbffb..506a63677 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -71,7 +71,7 @@ std::optional DrvInfo::queryDrvPath() const { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i == attrs->end()) drvPath = {std::nullopt}; else @@ -93,7 +93,7 @@ StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + NixStringContext context; if (i != attrs->end()) outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } @@ -124,7 +124,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; + NixStringContext context; outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 60bf45147..cea5b4202 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -38,17 +38,16 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; StringMap res; - for (auto & c_ : context) { + for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); }; - auto c = NixStringContextElem::parse(*store, c_); std::visit(overloaded { [&](const NixStringContextElem::Built & b) { drvs.push_back(DerivedPath::Built { @@ -112,7 +111,7 @@ struct RealisePathFlags { static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); @@ -158,7 +157,12 @@ static void mkOutputString( /* FIXME: we need to depend on the basic derivation, not derivation */ : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + NixStringContext { + NixStringContextElem::Built { + .drvPath = drvPath, + .output = o.first, + } + }); } /* Load and evaluate an expression from path specified by the @@ -178,17 +182,18 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v return storePath; }; - if (auto optStorePath = isValidDerivationInStore()) { - auto storePath = *optStorePath; - Derivation drv = state.store->readDerivation(storePath); + if (auto storePath = isValidDerivationInStore()) { + Derivation drv = state.store->readDerivation(*storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path2, {"=" + path2}); + attrs.alloc(state.sDrvPath).mkString(path2, { + NixStringContextElem::DrvDeep { .drvPath = *storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, storePath, drv, o); + mkOutputString(state, attrs, *storePath, drv, o); (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); } @@ -359,7 +364,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); - PathSet context; + NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", false, false).toOwned(); @@ -769,7 +774,7 @@ static RegisterPrimOp primop_abort({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); @@ -788,7 +793,7 @@ static RegisterPrimOp primop_throw({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); state.debugThrowLastTrace(ThrownError(s)); @@ -801,7 +806,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; + NixStringContext context; auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); @@ -1087,7 +1092,7 @@ drvName, Bindings * attrs, Value & v) Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; @@ -1233,8 +1238,7 @@ drvName, Bindings * attrs, Value & v) /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & c_ : context) { - auto c = NixStringContextElem::parse(*state.store, c_); + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every path in the dependency graph of the derivation (including @@ -1294,7 +1298,13 @@ drvName, Bindings * attrs, Value & v) auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(method, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput::CAFixed { @@ -1387,7 +1397,9 @@ drvName, Bindings * attrs, Value & v) } auto result = state.buildBindings(1 + drv.outputs.size()); - result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) mkOutputString(state, result, drvPath, drv, i); @@ -1432,7 +1444,7 @@ static RegisterPrimOp primop_placeholder({ /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); v.mkString(path.path.abs(), context); } @@ -1463,7 +1475,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; + NixStringContext context; auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so @@ -1478,7 +1490,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); + context.insert(NixStringContextElem::Opaque { .path = path2 }); v.mkString(path.abs(), context); } @@ -1534,7 +1546,7 @@ static RegisterPrimOp primop_pathExists({ following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), context); @@ -1556,12 +1568,12 @@ static RegisterPrimOp primop_baseNameOf({ of the argument. */ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; state.forceValue(*args[0], pos); if (args[0]->type() == nPath) { auto path = args[0]->path(); v.mkPath(path.path.isRoot() ? path : path.parent()); } else { + NixStringContext context; auto path = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false); @@ -1599,7 +1611,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V refsSink << s; refs = refsSink.getResultPaths(); } - auto context = state.store->printStorePathSet(refs); + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); + } v.mkString(s, context); } @@ -1630,7 +1647,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *i->value, context, "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", false, false).toOwned(); @@ -1781,7 +1798,7 @@ static RegisterPrimOp primop_readDir({ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1889,7 +1906,7 @@ static RegisterPrimOp primop_toXML({ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1939,22 +1956,23 @@ static RegisterPrimOp primop_fromJSON({ as an input by derivations. */ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') + for (auto c : context) { + if (auto p = std::get_if(&c)) + refs.insert(p->path); + else state.debugThrowLastTrace(EvalError({ .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", - name, path), + name, c.to_string()), .errPos = state.positions[pos] })); - refs.insert(state.store->parseStorePath(path)); } auto storePath = settings.readOnlyMode @@ -2055,7 +2073,7 @@ static void addPath( FileIngestionMethod method, const std::optional expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { // FIXME: handle CA derivation outputs (where path needs to @@ -2103,7 +2121,13 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = *expectedHash, + }, + .references = {}, + }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { StorePath dstPath = settings.readOnlyMode @@ -2123,7 +2147,7 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); @@ -2192,7 +2216,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; - PathSet context; + NixStringContext context; state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); @@ -3528,7 +3552,7 @@ static RegisterPrimOp primop_lessThan({ `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false); @@ -3567,7 +3591,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, { int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) @@ -3601,7 +3625,7 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3627,7 +3651,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; // discarded + NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); @@ -3673,7 +3697,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; @@ -3753,7 +3777,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); @@ -3850,7 +3874,7 @@ static RegisterPrimOp primop_split({ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); @@ -3890,15 +3914,15 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; + std::vector> to; to.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - PathSet ctx; + NixStringContext ctx; auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); to.emplace_back(s, std::move(ctx)); } - PathSet context; + NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db43e5771..07bf400cf 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } @@ -17,7 +17,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -33,17 +33,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto && p : context) { - auto c = NixStringContextElem::parse(*state.store, p); + NixStringContext context2; + for (auto && c : context) { if (auto * ptr = std::get_if(&c)) { - context2.emplace(state.store->printStorePath(ptr->drvPath)); + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); } else { /* Can reuse original item */ - context2.emplace(std::move(p)); + context2.emplace(std::move(c)); } } @@ -79,22 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, bool allOutputs = false; Strings outputs; }; - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); auto contextInfos = std::map(); - for (const auto & p : context) { - NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + for (auto && i : context) { std::visit(overloaded { - [&](NixStringContextElem::DrvDeep & d) { - contextInfos[d.drvPath].allOutputs = true; + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; }, - [&](NixStringContextElem::Built & b) { - contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output)); + [&](NixStringContextElem::Built && b) { + contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); }, - [&](NixStringContextElem::Opaque & o) { - contextInfos[o.path].path = true; + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; }, - }, ctx.raw()); + }, ((NixStringContextElem &&) i).raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -129,7 +129,7 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); @@ -143,13 +143,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .msg = hintfmt("context key '%s' is not a store path", name), .errPos = state.positions[i.pos] }); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(name)); + state.store->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) - context.emplace(name); + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } iter = i.value->attrs->find(sAllOutputs); @@ -161,7 +164,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); } - context.insert(concatStrings("=", name)); + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, + }); } } @@ -176,7 +181,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); - context.insert(concatStrings("!", outputName, "!", name)); + context.emplace(NixStringContextElem::Built { + .drvPath = namePath, + .output = std::string { outputName }, + }); } } } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 0dfa97fa3..4cf1f1e0b 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -18,7 +18,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg const auto & attrName = state.symbols[attr.name]; if (attrName == "fromPath") { - PathSet context; + NixStringContext context; fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } @@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg state.forceValue(*attr.value, attr.pos); toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; + NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } @@ -114,8 +114,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg }); } - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + state.mkStorePathString(*toPath, v); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c41bd60b6..2c0d98e74 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -13,7 +13,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a std::optional rev; std::optional ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -73,8 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0d0e00fa5..cd7039025 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -24,9 +24,8 @@ void emitTreeAttrs( auto attrs = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -107,7 +106,7 @@ static void fetchTree( const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -243,10 +242,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // early exit if pinned and already in the store if (expectedHash && expectedHash->type == htSHA256) { - auto expectedPath = - unpack - ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) - : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + auto expectedPath = state.store->makeFixedOutputPath( + name, + FixedOutputInfo { + .hash = { + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, + }, + .references = {} + }); if (state.store->isValidPath(expectedPath)) { state.allowAndSetStorePathString(expectedPath, v); diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 411bc0ac3..7586bdd9b 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -8,7 +8,7 @@ namespace nix { protected: std::string getJSONValue(Value& value) { std::stringstream ss; - PathSet ps; + NixStringContext ps; printValueAsJSON(state, true, value, noPos, ss, ps); return ss.str(); } diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 083359b7a..27d6920b0 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,69 +8,62 @@ namespace nix { -// Testing of trivial expressions -struct NixStringContextElemTest : public LibExprTest { - const Store & store() const { - return *LibExprTest::store; - } -}; - -TEST_F(NixStringContextElemTest, empty_invalid) { +TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), ""), + NixStringContextElem::parse(""), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, single_bang_invalid) { +TEST(NixStringContextElemTest, single_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!"), + NixStringContextElem::parse("!"), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, double_bang_invalid) { +TEST(NixStringContextElemTest, double_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!!/"), + NixStringContextElem::parse("!!/"), BadStorePath); } -TEST_F(NixStringContextElemTest, eq_slash_invalid) { +TEST(NixStringContextElemTest, eq_slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "=/"), + NixStringContextElem::parse("=/"), BadStorePath); } -TEST_F(NixStringContextElemTest, slash_invalid) { +TEST(NixStringContextElemTest, slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "/"), + NixStringContextElem::parse("/"), BadStorePath); } -TEST_F(NixStringContextElemTest, opaque) { - std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; - auto elem = NixStringContextElem::parse(store(), opaque); +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->path, store().parseStorePath(opaque)); - ASSERT_EQ(elem.to_string(store()), opaque); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); } -TEST_F(NixStringContextElemTest, drvDeep) { - std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), drvDeep); +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); - ASSERT_EQ(elem.to_string(store()), drvDeep); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); } -TEST_F(NixStringContextElemTest, built) { - std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), built); +TEST(NixStringContextElemTest, built) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); - ASSERT_EQ(elem.to_string(store()), built); + ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(elem.to_string(), built); } } @@ -116,12 +109,12 @@ Gen Arbitrary::arbitrary() namespace nix { -RC_GTEST_FIXTURE_PROP( +RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { - RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index d40a77302..4996a5bde 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,7 +11,7 @@ namespace nix { using json = nlohmann::json; json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); @@ -95,13 +95,13 @@ json printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore) const + NixStringContext & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 713356c7f..47ac90313 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { nlohmann::json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 4e87ac4b6..2539ad1c1 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,7 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos); @@ -32,7 +32,7 @@ static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; @@ -54,7 +54,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) { checkInterrupt(); @@ -166,7 +166,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -174,7 +174,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index ace7ead0f..6d702c0f2 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d524c4869..89c0c36fd 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -101,7 +101,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, @@ -113,13 +113,13 @@ class ExternalValueBase * Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore = true) const; + NixStringContext & context, bool copyToStore = true) const; /** * Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() @@ -269,9 +269,9 @@ public: void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); inline void mkString(const Symbol & s) { @@ -400,8 +400,6 @@ public: */ bool isTrivial() const; - NixStringContext getContext(const Store &); - auto listItems() { struct ListIterable diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 61d9c53df..f76fc76e4 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,11 +1,10 @@ #include "value/context.hh" -#include "store-api.hh" #include namespace nix { -NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +NixStringContextElem NixStringContextElem::parse(std::string_view s0) { std::string_view s = s0; @@ -25,41 +24,41 @@ NixStringContextElem NixStringContextElem::parse(const Store & store, std::strin "String content element beginning with '!' should have a second '!'"); } return NixStringContextElem::Built { - .drvPath = store.parseStorePath(s.substr(index + 1)), + .drvPath = StorePath { s.substr(index + 1) }, .output = std::string(s.substr(0, index)), }; } case '=': { return NixStringContextElem::DrvDeep { - .drvPath = store.parseStorePath(s.substr(1)), + .drvPath = StorePath { s.substr(1) }, }; } default: { return NixStringContextElem::Opaque { - .path = store.parseStorePath(s), + .path = StorePath { s }, }; } } } -std::string NixStringContextElem::to_string(const Store & store) const { +std::string NixStringContextElem::to_string() const { return std::visit(overloaded { [&](const NixStringContextElem::Built & b) { std::string res; res += '!'; res += b.output; res += '!'; - res += store.printStorePath(b.drvPath); + res += b.drvPath.to_string(); return res; }, [&](const NixStringContextElem::DrvDeep & d) { std::string res; res += '='; - res += store.printStorePath(d.drvPath); + res += d.drvPath.to_string(); return res; }, [&](const NixStringContextElem::Opaque & o) { - return store.printStorePath(o.path); + return std::string { o.path.to_string() }; }, }, raw()); } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 8719602d8..287ae08a9 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -26,8 +26,6 @@ public: } }; -class Store; - /** * Plain opaque path to some store object. * @@ -80,12 +78,15 @@ struct NixStringContextElem : _NixStringContextElem_Raw { using DrvDeep = NixStringContextElem_DrvDeep; using Built = NixStringContextElem_Built; - inline const Raw & raw() const { + inline const Raw & raw() const & { return static_cast(*this); } - inline Raw & raw() { + inline Raw & raw() & { return static_cast(*this); } + inline Raw && raw() && { + return static_cast(*this); + } /** * Decode a context string, one of: @@ -93,10 +94,10 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘=’ * - ‘!!’ */ - static NixStringContextElem parse(const Store & store, std::string_view s); - std::string to_string(const Store & store) const; + static NixStringContextElem parse(std::string_view s); + std::string to_string() const; }; -typedef std::vector NixStringContext; +typedef std::set NixStringContext; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c767e72e5..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + .references = {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e9686262a..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,15 +71,19 @@ DownloadFileResult downloadFile( dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + name, + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + .references = {}, + }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; - auto source = StringSource(sink.s); + auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 37664c065..56f47a4ac 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -20,16 +19,9 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include -#include - namespace nix { @@ -115,57 +107,6 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -177,16 +118,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif - - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - loadConfFile(); + initLibStore(); startSignalHandlerThread(); @@ -223,7 +155,10 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for @@ -236,16 +171,6 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - - preloadNSS(); - initLibStore(); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 751cf8c30..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -306,11 +306,22 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; return info; })->path; } @@ -414,15 +425,22 @@ StorePath BinaryCacheStore::addToStore( }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -434,7 +452,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; @@ -443,10 +461,16 @@ StorePath BinaryCacheStore::addTextToStore( dumpString(s, sink); StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, + }, + nar.first, + }; info.narSize = nar.second; - info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 27d1a1b6c..b7a56e791 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -83,16 +83,11 @@ struct BuildResult */ bool isNonDeterministic = false; - /** - * The derivation we built or the store path we substituted. - */ - DerivedPath path; - /** * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; /** * The start/stop times of the build (or one of the rounds, if it @@ -116,4 +111,15 @@ struct BuildResult } }; +/** + * A `BuildResult` together with its "primary key". + */ +struct KeyedBuildResult : BuildResult +{ + /** + * The derivation we built or the store path we substituted. + */ + DerivedPath path; +}; + } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 26faf8c8e..a4bb94b0e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -145,8 +145,20 @@ void DerivationGoal::work() void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { auto newWanted = wantedOutputs.union_(outputs); - if (!newWanted.isSubsetOf(wantedOutputs)) - needRestart = true; + switch (needRestart) { + case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: + if (!newWanted.isSubsetOf(wantedOutputs)) + needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed; + break; + case NeedRestartForMoreOutputs::OutputsAddedDoNeed: + /* No need to check whether we added more outputs, because a + restart is already queued up. */ + break; + case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed: + /* We are already building all outputs, so it doesn't matter if + we now want more. */ + break; + }; wantedOutputs = newWanted; } @@ -297,12 +309,29 @@ void DerivationGoal::outputsSubstitutionTried() In particular, it may be the case that the hole in the closure is an output of the current derivation, which causes a loop if retried. */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + { + bool substitutionFailed = + nrIncompleteClosure > 0 && + nrIncompleteClosure == nrFailed; + switch (retrySubstitution) { + case RetrySubstitution::NoNeed: + if (substitutionFailed) + retrySubstitution = RetrySubstitution::YesNeed; + break; + case RetrySubstitution::YesNeed: + // Should not be able to reach this state from here. + assert(false); + break; + case RetrySubstitution::AlreadyRetried: + debug("substitution failed again, but we already retried once. Not retrying again."); + break; + } + } nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - if (needRestart) { - needRestart = false; + if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { + needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; haveDerivation(); return; } @@ -330,6 +359,10 @@ void DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { + /* At this point we are building all outputs, so if more are wanted there + is no need to restart. */ + needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; + /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) @@ -451,8 +484,8 @@ void DerivationGoal::inputsRealised() return; } - if (retrySubstitution && !retriedSubstitution) { - retriedSubstitution = true; + if (retrySubstitution == RetrySubstitution::YesNeed) { + retrySubstitution = RetrySubstitution::AlreadyRetried; haveDerivation(); return; } @@ -570,8 +603,6 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - - buildResult = BuildResult { .path = buildResult.path }; } void DerivationGoal::started() @@ -982,7 +1013,7 @@ void DerivationGoal::resolvedFinished() auto resolvedDrv = *resolvedDrvGoal->drv; auto & resolvedResult = resolvedDrvGoal->buildResult; - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; if (resolvedResult.success()) { auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); @@ -1008,7 +1039,7 @@ void DerivationGoal::resolvedFinished() worker.store.printStorePath(drvPath), wantedOutput); auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + auto take1 = get(resolvedResult.builtOutputs, wantedOutput); if (take1) return *take1; /* The above `get` should work. But sateful tracking of @@ -1033,7 +1064,7 @@ void DerivationGoal::resolvedFinished() worker.store.registerDrvOutput(newRealisation); } outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + builtOutputs.emplace(wantedOutput, realisation); } runPostBuildHook( @@ -1158,7 +1189,7 @@ HookReply DerivationGoal::tryBuildHook() } -DrvOutputs DerivationGoal::registerOutputs() +SingleDrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1320,7 +1351,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() } -std::pair DerivationGoal::checkPathValidity() +std::pair DerivationGoal::checkPathValidity() { if (!drv->type().isPure()) return { false, {} }; @@ -1333,7 +1364,7 @@ std::pair DerivationGoal::checkPathValidity() return static_cast(names); }, }, wantedOutputs.raw()); - DrvOutputs validOutputs; + SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { auto initialOutput = get(initialOutputs, i.first); @@ -1376,7 +1407,7 @@ std::pair DerivationGoal::checkPathValidity() } } if (info.wanted && info.known && info.known->isValid()) - validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); + validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); } // If we requested all the outputs, we are always fine. @@ -1400,7 +1431,7 @@ std::pair DerivationGoal::checkPathValidity() } -DrvOutputs DerivationGoal::assertPathValidity() +SingleDrvOutputs DerivationGoal::assertPathValidity() { auto [allValid, validOutputs] = checkPathValidity(); if (!allValid) @@ -1411,7 +1442,7 @@ DrvOutputs DerivationGoal::assertPathValidity() void DerivationGoal::done( BuildResult::Status status, - DrvOutputs builtOutputs, + SingleDrvOutputs builtOutputs, std::optional ex) { buildResult.status = status; @@ -1452,12 +1483,28 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (waitee->buildResult.success()) - if (auto bfd = std::get_if(&waitee->buildResult.path)) - for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + if (!useDerivation) return; + auto & fullDrv = *dynamic_cast(drv.get()); + + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; + + auto outputs = fullDrv.inputDrvs.find(dg->drvPath); + if (outputs == fullDrv.inputDrvs.end()) return; + + for (auto & outputName : outputs->second) { + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = dg->drvPath, + .outputs = OutputsSpec::Names { outputName }, + }); + if (buildResult.success()) { + auto i = buildResult.builtOutputs.find(outputName); + if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { bfd->drvPath, output.outputName }, - realisation.outPath); + { dg->drvPath, outputName }, + i->second.outPath); + } + } } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 3a6f0c2d9..7033b7a58 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -78,22 +78,58 @@ struct DerivationGoal : public Goal */ std::map, StorePath> inputDrvOutputs; + /** + * See `needRestart`; just for that field. + */ + enum struct NeedRestartForMoreOutputs { + /** + * The goal state machine is progressing based on the current value of + * `wantedOutputs. No actions are needed. + */ + OutputsUnmodifedDontNeed, + /** + * `wantedOutputs` has been extended, but the state machine is + * proceeding according to its old value, so we need to restart. + */ + OutputsAddedDoNeed, + /** + * The goal state machine has progressed to the point of doing a build, + * in which case all outputs will be produced, so extensions to + * `wantedOutputs` no longer require a restart. + */ + BuildInProgressWillNotNeed, + }; + /** * Whether additional wanted outputs have been added. */ - bool needRestart = false; + NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + + /** + * See `retrySubstitution`; just for that field. + */ + enum RetrySubstitution { + /** + * No issues have yet arose, no need to restart. + */ + NoNeed, + /** + * Something failed and there is an incomplete closure. Let's retry + * substituting. + */ + YesNeed, + /** + * We are current or have already retried substitution, and whether or + * not something goes wrong we will not retry again. + */ + AlreadyRetried, + }; /** * Whether to retry substituting the outputs after building the * inputs. This is done in case of an incomplete closure. */ - bool retrySubstitution = false; - - /** - * Whether we've retried substitution, in which case we won't try - * again. - */ - bool retriedSubstitution = false; + RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed; /** * The derivation stored at drvPath. @@ -217,7 +253,7 @@ struct DerivationGoal : public Goal * Check that the derivation outputs all exist and register them * as valid. */ - virtual DrvOutputs registerOutputs(); + virtual SingleDrvOutputs registerOutputs(); /** * Open a log file and a pipe to it. @@ -270,17 +306,17 @@ struct DerivationGoal : public Goal * Update 'initialOutputs' to determine the current status of the * outputs of the derivation. Also returns a Boolean denoting * whether all outputs are valid and non-corrupt, and a - * 'DrvOutputs' structure containing the valid and wanted + * 'SingleDrvOutputs' structure containing the valid and wanted * outputs. */ - std::pair checkPathValidity(); + std::pair checkPathValidity(); /** * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'DrvOutputs' structure containing the wanted + * returns a 'SingleDrvOutputs' structure containing the wanted * outputs. */ - DrvOutputs assertPathValidity(); + SingleDrvOutputs assertPathValidity(); /** * Forcibly kill the child process, if any. @@ -293,7 +329,7 @@ struct DerivationGoal : public Goal void done( BuildResult::Status status, - DrvOutputs builtOutputs = {}, + SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); void waiteeDone(GoalPtr waitee, ExitCode result) override; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 2925fe3ca..74eae0692 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -10,16 +10,8 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); @@ -47,7 +39,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } } -std::vector Store::buildPathsWithResults( +std::vector Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) @@ -55,23 +47,23 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); + std::vector> state; + + for (const auto & req : reqs) { + auto goal = worker.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); } worker.run(goals); - std::vector results; + std::vector results; - for (auto & i : goals) - results.push_back(i->buildResult); + for (auto & [req, goalPtr] : state) + results.emplace_back(KeyedBuildResult { + goalPtr->getBuildResult(req), + /* .path = */ req, + }); return results; } @@ -84,15 +76,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); - return goal->buildResult; + return goal->getBuildResult(DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All {}, + }); } catch (Error & e) { return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, }; }; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index d59b94797..ca7097a68 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,6 +11,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } +BuildResult Goal::getBuildResult(const DerivedPath & req) { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + void addToWeakGoals(WeakGoals & goals, GoalPtr p) { if (goals.find(p) != goals.end()) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index f4bf6f38b..c0e12a2ed 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -81,11 +81,26 @@ struct Goal : public std::enable_shared_from_this */ ExitCode exitCode = ecBusy; +protected: /** * Build result. */ BuildResult buildResult; +public: + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &); + /** * Exception containing an error message, if any. */ @@ -93,7 +108,6 @@ struct Goal : public std::enable_shared_from_this Goal(Worker & worker, DerivedPath path) : worker(worker) - , buildResult { .path = std::move(path) } { } virtual ~Goal() diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 58d6901d3..21cd6e7ee 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1335,7 +1335,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo result.rethrow(); } - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override @@ -2174,7 +2174,7 @@ void LocalDerivationGoal::runChild() } -DrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2395,27 +2395,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair res { - false, - {}, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.first = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.second.insert(r); + res.others.insert(r); } } return res; @@ -2448,18 +2447,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); - - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (*scratchPath != finalPath) { + ValidPathInfo newInfo0 { + worker.store, + outputPathName(drv->name, outputName), + FixedOutputInfo { + .hash = { + .method = outputHash.method, + .hash = got, + }, + .references = rewriteRefs(), + }, + Hash::dummy, + }; + if (*scratchPath != newInfo0.path) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); dumpPath(actualPath, rsink2); rsink2.flush(); }); @@ -2470,19 +2473,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } HashResult narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, - }; - + newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -2504,8 +2496,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) + newInfo0.references = std::move(refs.others); + if (refs.self) newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2519,7 +2511,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Check wanted hash */ const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ @@ -2691,7 +2683,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { auto oldinfo = get(initialOutputs, outputName); @@ -2710,7 +2702,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store.registerDrvOutput(thisRealisation); } if (wantedOutputs.contains(outputName)) - builtOutputs.emplace(thisRealisation.id, thisRealisation); + builtOutputs.emplace(outputName, thisRealisation); } return builtOutputs; diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 42d32a31a..9acd7593d 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -237,7 +237,7 @@ struct LocalDerivationGoal : public DerivationGoal * Check that the derivation outputs all exist and register them * as valid. */ - DrvOutputs registerOutputs() override; + SingleDrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2af105b4d..190fb455a 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,7 +95,9 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f775f8486..6ad4a0e2b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -92,6 +92,7 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } + std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; @@ -104,6 +105,20 @@ std::shared_ptr Worker::makeDrvOutputSubstitutionGoal return goal; } + +GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) -> GoalPtr { + return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + }, + [&](const DerivedPath::Opaque & bo) -> GoalPtr { + return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + }, + }, req.raw()); +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 48a1a27fa..bb51d641d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -181,7 +181,7 @@ public: */ /** - * derivation goal + * @ref DerivationGoal "derivation goal" */ private: std::shared_ptr makeDerivationGoalCommon( @@ -196,11 +196,19 @@ public: const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); /** - * substitution goal + * @ref SubstitutionGoal "substitution goal" */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + /** * Remove a dead goal. */ diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index cf32ccdc4..055b216db 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -21,39 +21,35 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) -{ - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); -} - -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { - return "text:" + th.hash.to_string(Base32, true); + [](const TextHash & th) { + return "text:" + + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + [](const FixedOutputHash & fsh) { + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextHashMethod & th) { + [](const TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod & fshm) { + [](const FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { @@ -97,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); @@ -115,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), }); }, - }, caMethod); + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -126,26 +122,55 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +const Hash & ContentAddress::getHash() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; - } - }, ca); + }, + }, raw); +} + +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](const TextHash & h) -> ContentAddressWithReferences { + return TextInfo { + .hash = h, + .references = {}, + }; + }, + [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + return FixedOutputInfo { + .hash = h, + .references = {}, + }; + }, + }, ca.raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 368a7ec48..2f98950fb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,12 +3,30 @@ #include #include "hash.hh" +#include "path.hh" #include "comparator.hh" namespace nix { +/* + * Content addressing method + */ + +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ + /** - * An enumeration of the ways we can serialize file system objects. + * The single way we can serialize "text" file system objects. + * + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ +struct TextHashMethod : std::monostate { }; + +/** + * An enumeration of the main ways we can serialize file system + * objects. */ enum struct FileIngestionMethod : uint8_t { /** @@ -22,6 +40,53 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ +std::string makeFileIngestionPrefix(FileIngestionMethod m); + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; + + GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); +}; + +/** + * An enumeration of all the ways we can serialize file system objects. + * + * Just the type of a content address. Combine with the hash itself, and + * we have a `ContentAddress` as defined below. Combine that, in turn, + * with info on references, and we have `ContentAddressWithReferences`, + * as defined further below. + */ +struct ContentAddressMethod +{ + typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > Raw; + + Raw raw; + + GENERATE_CMP(ContentAddressMethod, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward(arg)...) + { } + + static ContentAddressMethod parse(std::string_view rawCaMethod); + + std::string render() const; +}; + + +/* + * Mini content address + */ + /** * Somewhat obscure, used by \ref Derivation derivations and * `builtins.toFile` currently. @@ -36,7 +101,7 @@ struct TextHash { }; /** - * For path computed by makeFixedOutputPath. + * Used by most store objects that are content-addressed. */ struct FixedOutputHash { /** @@ -65,41 +130,96 @@ struct FixedOutputHash { * - ‘fixed:::’: For paths computed by * Store::makeFixedOutputPath() / Store::addToStore(). */ -typedef std::variant< - TextHash, - FixedOutputHash -> ContentAddress; +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; -/** - * Compute the prefix to the hash algorithm which indicates how the - * files were ingested. - */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); + Raw raw; -/** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). - */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + GENERATE_CMP(ContentAddress, me->raw); -std::string renderContentAddress(ContentAddress ca); + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + + const Hash & getHash() const; +}; std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); - -std::optional parseContentAddressOpt(std::string_view rawCaOpt); - -Hash getContentAddressHash(const ContentAddress & ca); /* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +/** + * A set of references to other store objects. + * + * References to other store objects are tracked with store paths, self + * references however are tracked with a boolean. + */ +struct StoreReferences { + /** + * References to other store objects + */ + StorePathSet others; + + /** + * Reference to this store object + */ + bool self = false; + + /** + * @return true iff no references, i.e. others is empty and self is + * false. + */ + bool empty() const; + + /** + * Returns the numbers of references, i.e. the size of others + 1 + * iff self is true. + */ + size_t size() const; + + GENERATE_CMP(StoreReferences, me->self, me->others); +}; + +// This matches the additional info that we need for makeTextPath +struct TextInfo { + TextHash hash; + /** + * References to other store objects only; self references + * disallowed + */ + StorePathSet references; + + GENERATE_CMP(TextInfo, me->hash, me->references); +}; + +struct FixedOutputInfo { + FixedOutputHash hash; + /** + * References to other store objects or this one. + */ + StoreReferences references; + + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; /** @@ -107,13 +227,27 @@ struct FixedOutputHashMethod { * * A ContentAddress without a Hash. */ -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + Raw raw; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); + GENERATE_CMP(ContentAddressWithReferences, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Create a ContentAddressWithReferences from a mere ContentAddress, by + * assuming no references in all cases. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &); +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 63898f8dc..af9a76f1e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { + [&](const FixedOutputHashMethod & fohm) { auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -637,7 +637,10 @@ static void performOp(TunnelLogger * logger, ref store, to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - worker_proto::write(*store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, to, builtOutputs); } break; } @@ -880,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref store, info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; @@ -1064,6 +1067,8 @@ void processConnection( opCount++; + debug("performing daemon worker op: %d", op); + try { performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7eb5cd275..15f3908ed 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,8 +36,8 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } @@ -942,7 +942,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const envHasRightPath(doia.path, i.first); }, [&](const DerivationOutput::CAFixed & dof) { - StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](const DerivationOutput::CAFloating &) { @@ -989,7 +989,8 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & _json) + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) { std::set keys; auto json = (std::map) _json; @@ -1028,6 +1029,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::CAFloating { .method = method, @@ -1040,6 +1042,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::Impure { .method = method, diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index ccdde36ca..d00b23b6d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -136,11 +136,15 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + /** + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..4c66d08ee 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,12 +7,23 @@ #include #include +#include #include #include #include #include +#include + +#ifdef __GLIBC__ +#include +#include +#include +#endif + +#include "config-impl.hh" + namespace nix { @@ -41,7 +52,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -185,18 +195,18 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str, bool append) +template<> SandboxMode BaseSetting::parse(const std::string & str) const { - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; + if (str == "true") return smEnabled; + else if (str == "relaxed") return smRelaxed; + else if (str == "false") return smDisabled; else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> bool BaseSetting::isAppendable() +template<> struct BaseSetting::trait { - return false; -} + static constexpr bool appendable = false; +}; template<> std::string BaseSetting::to_string() const { @@ -228,23 +238,23 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str, bool append) +unsigned int MaxBuildJobsSetting::parse(const std::string & str) const { - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } } -void PluginFilesSetting::set(const std::string & str, bool append) +Paths PluginFilesSetting::parse(const std::string & str) const { if (pluginsLoaded) throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - BaseSetting::set(str, append); + return BaseSetting::parse(str); } @@ -281,6 +291,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -291,6 +337,24 @@ void assertLibStoreInitialized() { } void initLibStore() { + + initLibUtil(); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + loadConfFile(); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f598ed4a8..609cf53b8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -26,7 +26,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + unsigned int parse(const std::string & str) const override; }; struct PluginFilesSetting : public BaseSetting @@ -43,7 +43,7 @@ struct PluginFilesSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Paths parse(const std::string & str) const override; }; const uint32_t maxIdsPerBuild = @@ -458,11 +458,6 @@ public: )", {"env-keep-derivations"}}; - /** - * Whether to lock the Nix client and worker to the same CPU. - */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d2ddbbe5f..2012584e0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings(conn->from); auto s = readString(conn->from); @@ -287,19 +287,18 @@ public: conn->to.flush(); - BuildResult status { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult status; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return status; } @@ -330,7 +329,7 @@ public: conn->to.flush(); - BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; + BuildResult result; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c3be1b461..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -710,6 +710,7 @@ void canonicalisePathMetaData(const Path & path, canonicalisePathMetaData(path, uidRange, inodesSeen); } + void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -888,7 +889,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1221,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { + if (auto foHash = std::get_if(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1234,7 +1235,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if(&*info.ca)) { + if (auto textHash = std::get_if(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1320,7 +1321,19 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name, references); + ContentAddressWithReferences desc = FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1340,7 +1353,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1364,10 +1377,13 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; - info.references = references; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1384,7 +1400,10 @@ StorePath LocalStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 64d172918..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,18 +27,17 @@ std::map makeContentAddressed( StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + StoreReferences refs; for (auto & ref : oldInfo->references) { if (ref == path) - hasSelfReference = true; + refs.self = true; else { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } @@ -49,24 +48,28 @@ std::map makeContentAddressed( auto narModuloHash = hashModuloSink.finish().first; - auto dstPath = dstStore.makeFixedOutputPath( - FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + ValidPathInfo info { + dstStore, + path.name(), + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + .references = std::move(refs), + }, + Hash::dummy, + }; - printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); StringSink sink2; - RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2); rsink2(sink.s); rsink2.flush(); - ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.narHash = hashString(htSHA256, sink2.s); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }; StringSource source(sink2.s); dstStore.addToStore(info, source); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 2645f468b..c7176d30f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -273,7 +273,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..d17253741 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,18 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { - auto corrupt = [&]() { - return Error("NAR info file '%1%' is corrupt", whence); + unsigned line = 1; + + auto corrupt = [&](const char * reason) { + return Error("NAR info file '%1%' is corrupt: %2%", whence, + std::string(reason) + (line > 0 ? " at line " + std::to_string(line) : "")); }; auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - throw corrupt(); + throw corrupt("bad hash"); } }; @@ -26,12 +29,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); + if (colon == std::string::npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); + if (eol == std::string::npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -47,7 +50,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & fileHash = parseHashField(value); else if (name == "FileSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid FileSize"); fileSize = *n; } else if (name == "NarHash") { @@ -56,12 +59,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } else if (name == "NarSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid NarSize"); narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) throw corrupt(); + if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); } @@ -72,17 +75,26 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) throw corrupt(); + if (ca) throw corrupt("extra CA"); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; + line += 1; } if (compression == "") compression = "bzip2"; - if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) { + line = 0; // don't include line information in the error + throw corrupt( + !havePath ? "StorePath missing" : + !haveNarHash ? "NarHash missing" : + url.empty() ? "URL missing" : + narSize == 0 ? "NarSize missing or zero" + : "?"); + } } std::string NarInfo::to_string(const Store & store) const diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 3cae8e659..5dbdafac3 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index bd55a9d06..e60d7abe0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,25 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } - -bool ValidPathInfo::isContentAddressed(const Store & store) const +std::optional ValidPathInfo::contentAddressWithReferences() const { - if (! ca) return false; + if (! ca) + return std::nullopt; - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + .hash = th, + .references = references, + }; }, - [&](const FixedOutputHash & fsh) { + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); + return FixedOutputInfo { + .hash = foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, ca->raw); +} + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto fullCaOpt = contentAddressWithReferences(); + + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); bool res = caPath == path; @@ -77,6 +97,29 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + std::string_view name, + ContentAddressWithReferences && ca, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(name, ca)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); + }, + [this](FixedOutputInfo && foi) { + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); + this->ca = std::move((FixedOutputHash &&) foi); + }, + }, std::move(ca).raw); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -93,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned if (format >= 16) { source >> info.ultimate; info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 9af1309a0..221523622 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -92,6 +92,13 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ + std::optional contentAddressWithReferences() const; + /** * @return true iff the path is verifiably content-addressed. */ @@ -118,6 +125,9 @@ struct ValidPathInfo ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + virtual ~ValidPathInfo() { } static ValidPathInfo read(Source & source, const Store & store, unsigned int format); diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index a18cf2aa8..3922d1267 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -13,9 +13,25 @@ namespace nix { class Store; +/** + * A general `Realisation` key. + * + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. + */ struct DrvOutput { - // The hash modulo of the derivation + /** + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. + */ Hash drvHash; + + /** + * The name of the output. + */ std::string outputName; std::string to_string() const; @@ -60,6 +76,21 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +/** + * Collection type for a single derivation's outputs' `Realisation`s. + * + * Since these are the outputs of a single derivation, we know the + * output names are unique so we can use them as the map key. + */ +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ typedef std::map DrvOutputs; struct OpaquePath { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b862902d1..a6e8b9577 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -78,7 +78,7 @@ void write(const Store & store, Sink & out, const std::optional & o ContentAddress read(const Store & store, Source & from, Phantom _) { - return parseContentAddress(readString(from)); + return ContentAddress::parse(readString(from)); } void write(const Store & store, Sink & out, const ContentAddress & ca) @@ -125,10 +125,26 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput) } -BuildResult read(const Store & store, Source & from, Phantom _) +KeyedBuildResult read(const Store & store, Source & from, Phantom _) { auto path = worker_proto::read(store, from, Phantom {}); - BuildResult res { .path = path }; + auto br = worker_proto::read(store, from, Phantom {}); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void write(const Store & store, Sink & to, const KeyedBuildResult & res) +{ + worker_proto::write(store, to, res.path); + worker_proto::write(store, to, static_cast(res)); +} + + +BuildResult read(const Store & store, Source & from, Phantom _) +{ + BuildResult res; res.status = (BuildResult::Status) readInt(from); from >> res.errorMsg @@ -136,13 +152,16 @@ BuildResult read(const Store & store, Source & from, Phantom _) >> res.isNonDeterministic >> res.startTime >> res.stopTime; - res.builtOutputs = worker_proto::read(store, from, Phantom {}); + auto builtOutputs = worker_proto::read(store, from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); return res; } void write(const Store & store, Sink & to, const BuildResult & res) { - worker_proto::write(store, to, res.path); to << res.status << res.errorMsg @@ -150,7 +169,10 @@ void write(const Store & store, Sink & to, const BuildResult & res) << res.isNonDeterministic << res.startTime << res.stopTime; - worker_proto::write(store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(store, to, builtOutputs); } @@ -168,7 +190,7 @@ void write(const Store & store, Sink & out, const std::optional & sto std::optional read(const Store & store, Source & from, Phantom> _) { - return parseContentAddressOpt(readString(from)); + return ContentAddress::parseOpt(readString(from)); } void write(const Store & store, Sink & out, const std::optional & caOpt) @@ -586,7 +608,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << caMethod.render(); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -644,7 +666,7 @@ ref RemoteStore::addCAToStore( } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -865,7 +887,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( +std::vector RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) @@ -880,7 +902,7 @@ std::vector RemoteStore::buildPathsWithResults( writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return worker_proto::read(*this, conn->from, Phantom> {}); } else { // Avoid deadlock. conn_.reset(); @@ -889,21 +911,25 @@ std::vector RemoteStore::buildPathsWithResults( // fails, but meh. buildPaths(paths, buildMode, evalStore); - std::vector results; + std::vector results; for (auto & path : paths) { std::visit( overloaded { [&](const DerivedPath::Opaque & bo) { - results.push_back(BuildResult { - .status = BuildResult::Substituted, - .path = bo, + results.push_back(KeyedBuildResult { + { + .status = BuildResult::Substituted, + }, + /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - BuildResult res { - .status = BuildResult::Built, - .path = bfd, + KeyedBuildResult res { + { + .status = BuildResult::Built + }, + /* .path = */ bfd, }; OutputPathMap outputs; @@ -922,10 +948,10 @@ std::vector RemoteStore::buildPathsWithResults( queryRealisation(outputId); if (!realisation) throw MissingRealisation(outputId); - res.builtOutputs.emplace(realisation->id, *realisation); + res.builtOutputs.emplace(output, *realisation); } else { res.builtOutputs.emplace( - outputId, + output, Realisation { .id = outputId, .outPath = outputPath, @@ -952,12 +978,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult res; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { @@ -965,7 +986,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - res.builtOutputs = builtOutputs; + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return res; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1c45f543e..a30466647 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -114,7 +114,7 @@ public: void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fc3ed46a..5bee1af9f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" @@ -98,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). is the name of the output (usually, "out"). -

= base-16 representation of a SHA-256 hash of: +

= base-16 representation of a SHA-256 hash of + + = if = "text:...": the string written to the resulting store path - if = "source": + if = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if = "output:": @@ -162,63 +165,63 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const StoreReferences & references) { - for (auto & i : references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { - assert(references.empty()); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), + info.hash.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { - [&](const TextHash & th) { - return makeTextPath(name, th.hash, references); + [&](const TextInfo & ti) { + return makeTextPath(name, ti); }, - [&](const FixedOutputHash & fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](const FixedOutputInfo & foi) { + return makeFixedOutputPath(name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, ca.raw); } @@ -228,7 +231,14 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -237,7 +247,10 @@ StorePath Store::computeStorePathForText( std::string_view s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -425,11 +438,18 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = {}, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -521,7 +541,9 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -538,10 +560,11 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -1025,7 +1048,9 @@ void copyStorePath( // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->path.name(), + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1137,7 +1162,9 @@ std::map copyPaths( auto storePathForSrc = currentPathInfo.path; auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + storePathForDst = dstStore.makeFixedOutputPathFromCA( + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5bee272bf..c910d1c96 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -92,6 +92,7 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; +struct KeyedBuildResult; typedef std::map> StorePathCAMap; @@ -268,17 +269,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** * Preparatory part of addToStore(). @@ -575,7 +570,7 @@ public: * case of a build/substitution error, this function won't throw an * exception, but return a BuildResult containing an error message. */ - virtual std::vector buildPathsWithResults( + virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 80ee52fd0..6f94904dd 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -1,6 +1,7 @@ #include #include +#include "experimental-features.hh" #include "derivations.hh" #include "tests/libstore.hh" @@ -9,10 +10,32 @@ namespace nix { class DerivationTest : public LibStoreTest { +public: + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; }; -#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ STR ## _json, \ @@ -22,7 +45,7 @@ class DerivationTest : public LibStoreTest OUTPUT_NAME)); \ } \ \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ DerivationOutput { VAL }, \ @@ -30,10 +53,11 @@ class DerivationTest : public LibStoreTest *store, \ DRV_NAME, \ OUTPUT_NAME, \ - STR ## _json)); \ + STR ## _json, \ + mockXpSettings)); \ } -TEST_JSON(inputAddressed, +TEST_JSON(DerivationTest, inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -42,7 +66,7 @@ TEST_JSON(inputAddressed, }), "drv-name", "output-name") -TEST_JSON(caFixed, +TEST_JSON(DerivationTest, caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -56,7 +80,7 @@ TEST_JSON(caFixed, }), "drv-name", "output-name") -TEST_JSON(caFloating, +TEST_JSON(CaDerivationTest, caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -66,12 +90,12 @@ TEST_JSON(caFloating, }), "drv-name", "output-name") -TEST_JSON(deferred, +TEST_JSON(DerivationTest, deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(impure, +TEST_JSON(ImpureDerivationTest, impure, R"({ "hashAlgo": "r:sha256", "impure": true diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c7a6f8688..34b2fc17b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, KeyedBuildResult); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh new file mode 100644 index 000000000..b6cae5ec3 --- /dev/null +++ b/src/libutil/config-impl.hh @@ -0,0 +1,71 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * One only needs to include this when one is declaring a + * `BaseClass` setting, or as derived class of such an + * instantiation. + */ + +#include "config.hh" + +namespace nix { + +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template +bool BaseSetting::isAppendable() +{ + return trait::appendable; +} + +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); + +template +void BaseSetting::appendOrSet(T && newValue, bool append) +{ + static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + assert(!append); + value = std::move(newValue); +} + +template +void BaseSetting::set(const std::string & str, bool append) +{ + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) + appendOrSet(parse(str), append); + else { + assert(experimentalFeature); + warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", + name, + showExperimentalFeature(*experimentalFeature)); + } +} + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index a42f3a849..085a884dc 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,8 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "config-impl.hh" + #include namespace nix { @@ -80,6 +82,8 @@ void Config::getSettings(std::map & res, bool overridd void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { unsigned int pos = 0; + std::vector> parsedContents; + while (pos < contents.size()) { std::string line; while (pos < contents.size() && contents[pos] != '\n') @@ -125,8 +129,21 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string auto i = tokens.begin(); advance(i, 2); - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + parsedContents.push_back({ + name, + concatStringsSep(" ", Strings(i, tokens.end())), + }); }; + + // First apply experimental-feature related settings + for (auto & [name, value] : parsedContents) + if (name == "experimental-features" || name == "extra-experimental-features") + set(name, value); + + // Then apply other settings + for (auto & [name, value] : parsedContents) + if (name != "experimental-features" && name != "extra-experimental-features") + set(name, value); } void AbstractConfig::applyConfigFile(const Path & path) @@ -202,12 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -bool BaseSetting::isAppendable() -{ - return false; -} - template void BaseSetting::convertToArg(Args & args, const std::string & category) { @@ -231,9 +242,9 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> std::string BaseSetting::parse(const std::string & str) const { - value = str; + return str; } template<> std::string BaseSetting::to_string() const @@ -242,11 +253,11 @@ template<> std::string BaseSetting::to_string() const } template -void BaseSetting::set(const std::string & str, bool append) +T BaseSetting::parse(const std::string & str) const { static_assert(std::is_integral::value, "Integer required."); if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("setting '%s' has invalid value '%s'", name, str); } @@ -258,12 +269,12 @@ std::string BaseSetting::to_string() const return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> bool BaseSetting::parse(const std::string & str) const { if (str == "true" || str == "yes" || str == "1") - value = true; + return true; else if (str == "false" || str == "no" || str == "0") - value = false; + return false; else throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } @@ -291,16 +302,15 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> Strings BaseSetting::parse(const std::string & str) const { - auto ss = tokenizeString(str); - if (!append) value.clear(); - for (auto & s : ss) value.push_back(std::move(s)); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) value.push_back(std::move(s)); } template<> std::string BaseSetting::to_string() const @@ -308,16 +318,16 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringSet BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) - value.insert(s); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting::to_string() const @@ -325,21 +335,24 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting>::set(const std::string & str, bool append) +template<> std::set BaseSetting>::parse(const std::string & str) const { - if (!append) value.clear(); + std::set res; for (auto & s : tokenizeString(str)) { auto thisXpFeature = parseExperimentalFeature(s); if (thisXpFeature) - value.insert(thisXpFeature.value()); + res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); } + return res; } -template<> bool BaseSetting>::isAppendable() +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting>::to_string() const @@ -350,20 +363,23 @@ template<> std::string BaseSetting>::to_string() c return concatStringsSep(" ", stringifiedXpFeatures); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringMap BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); + StringMap res; for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) - value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } + return res; } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && [k, v] : std::move(newValue)) + value.emplace(std::move(k), std::move(v)); } template<> std::string BaseSetting::to_string() const @@ -387,15 +403,15 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; -void PathSetting::set(const std::string & str, bool append) +Path PathSetting::parse(const std::string & str) const { if (str == "") { if (allowEmpty) - value = ""; + return ""; else throw UsageError("setting '%s' cannot be empty", name); } else - value = canonPath(str); + return canonPath(str); } bool GlobalConfig::set(const std::string & name, const std::string & value) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 162626791..2675baed7 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -215,8 +215,11 @@ protected: virtual void set(const std::string & value, bool append = false) = 0; - virtual bool isAppendable() - { return false; } + /** + * Whether the type is appendable; i.e. whether the `append` + * parameter to `set()` is allowed to be `true`. + */ + virtual bool isAppendable() = 0; virtual std::string to_string() const = 0; @@ -241,6 +244,23 @@ protected: const T defaultValue; const bool documentDefault; + /** + * Parse the string into a `T`. + * + * Used by `set()`. + */ + virtual T parse(const std::string & str) const; + + /** + * Append or overwrite `value` with `newValue`. + * + * Some types to do not support appending in which case `append` + * should never be passed. The default handles this case. + * + * @param append Whether to append or overwrite. + */ + virtual void appendOrSet(T && newValue, bool append); + public: BaseSetting(const T & def, @@ -268,9 +288,25 @@ public: template void setDefault(const U & v) { if (!overridden) value = v; } - void set(const std::string & str, bool append = false) override; + /** + * Require any experimental feature the setting depends on + * + * Uses `parse()` to get the value from `str`, and `appendOrSet()` + * to set it. + */ + void set(const std::string & str, bool append = false) override final; - bool isAppendable() override; + /** + * C++ trick; This is template-specialized to compile-time indicate whether + * the type is appendable. + */ + struct trait; + + /** + * Always defined based on the C++ magic + * with `trait` above. + */ + bool isAppendable() override final; virtual void override(const T & v) { @@ -336,7 +372,7 @@ public: options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Path parse(const std::string & str) const override; Path operator +(const char * p) const { return value + p; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 5b4418714..bd1899662 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -189,6 +189,16 @@ constexpr std::array xpFeatureDetails = {{ runtime dependencies. )", }, + { + .tag = Xp::DaemonTrustOverride, + .name = "daemon-trust-override", + .description = R"( + Allow forcing trusting or not trusting clients with + `nix-daemon`. This is useful for testing, but possibly also + useful for various experiments with `nix-daemon --stdio` + networking. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 8ef66263a..3c00bc4e5 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -28,6 +28,7 @@ enum struct ExperimentalFeature AutoAllocateUids, Cgroups, DiscardReferences, + DaemonTrustOverride, }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 0c8f5727d..2c36d9d94 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -16,7 +17,6 @@ namespace nix { - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index f250e934e..886e70da5 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -82,6 +82,7 @@ namespace nix { TestSetting() : AbstractSetting("test", "test", {}) {} void set(const std::string & value, bool append) override {} std::string to_string() const override { return {}; } + bool isAppendable() override { return false; } }; Config config; @@ -90,6 +91,7 @@ namespace nix { ASSERT_FALSE(config.set("test", "value")); config.addSetting(&setting); ASSERT_TRUE(config.set("test", "value")); + ASSERT_FALSE(config.set("extra-test", "value")); } TEST(Config, withInitialValue) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 843a10eab..21d1c8dcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak)); namespace nix { +void initLibUtil() { +} + std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); @@ -1744,13 +1747,39 @@ void triggerInterrupt() } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} void startSignalHandlerThread() { updateWindowSize(); - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); + saveSignalMask(); sigset_t set; sigemptyset(&set); @@ -1767,6 +1796,20 @@ void startSignalHandlerThread() static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 85ab77b1b..040fed68f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,6 +32,7 @@ namespace nix { struct Sink; struct Source; +void initLibUtil(); /** * The system for which Nix is compiled. @@ -445,6 +446,8 @@ void setStackSize(size_t stackSize); /** * Restore the original inherited Unix process context (such as signal * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). */ void restoreProcessContext(bool restoreMounts = true); @@ -814,9 +817,26 @@ class Callback; /** * Start a thread that handles various signals. Also block those signals * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0081bab25..8b3f903f6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -962,7 +962,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); metaObj[j] = nullptr; } else { - PathSet context; + NixStringContext context; metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index f70381bf2..9e916abc4 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -119,9 +119,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ auto attrs = state.buildBindings(3); - attrs.alloc("manifest").mkString( - state.store->printStorePath(manifestFile), - {state.store->printStorePath(manifestFile)}); + state.mkStorePathString(manifestFile, attrs.alloc("manifest")); attrs.insert(state.symbols.create("derivations"), &manifest); Value args; args.mkAttrs(attrs); @@ -132,7 +130,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); - PathSet context; + NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 2b381288a..fa6cc2bd7 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -43,7 +43,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v, [&]() { return v.determinePos(noPos); }); - PathSet context; + NixStringContext context; if (evalOnly) { Value vRes; if (autoArgs.empty()) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7035e6a7b..40f30eb63 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -204,10 +204,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -218,7 +218,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string hash = *i++; std::string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + .references = {}, + }))); } @@ -935,7 +941,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - worker_proto::write(*store, out, status.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, out, builtOutputs); } break; @@ -964,7 +973,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5168413d2..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -42,14 +42,18 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + std::move(*namePart), + FixedOutputInfo { + .hash = { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + .references = {}, + }, narHash, }; info.narSize = sink.s.size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource(sink.s); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 57c355f0c..bcc00d490 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -98,7 +98,7 @@ struct CmdBundle : InstallableValueCommand if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - PathSet context2; + NixStringContext context2; auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); auto attr2 = vRes->attrs->get(evalState->sOutPath); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7ae7b4ea6..c1a91c63d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -273,8 +273,12 @@ static std::pair authPeer(const PeerInfo & peer) /** * Run a server. The loop opens a socket and accepts new connections from that * socket. + * + * @param forceTrustClientOpt If present, force trusting or not trusted + * the client. Otherwise, decide based on the authentication settings + * and user credentials (from the unix domain socket). */ -static void daemonLoop() +static void daemonLoop(std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -317,9 +321,18 @@ static void daemonLoop() closeOnExec(remote.get()); - PeerInfo peer = getPeerInfo(remote.get()); - auto [_trusted, user] = authPeer(peer); - auto trusted = _trusted; + PeerInfo peer { .pidKnown = false }; + TrustedFlag trusted; + std::string user; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = getPeerInfo(remote.get()); + auto [_trusted, _user] = authPeer(peer); + trusted = _trusted; + user = _user; + }; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -410,38 +423,47 @@ static void forwardStdioConnection(RemoteStore & store) { * Unlike `forwardStdioConnection()` we do process commands ourselves in * this case, not delegating to another daemon. * - * @note `Trusted` is unconditionally passed because in this mode we - * blindly trust the standard streams. Limiting access to those is - * explicitly not `nix-daemon`'s responsibility. + * @param trustClient Whether to trust the client. Forwarded directly to + * `processConnection()`. */ -static void processStdioConnection(ref store) +static void processStdioConnection(ref store, TrustedFlag trustClient) { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(store, from, to, Trusted, NotRecursive); + processConnection(store, from, to, trustClient, NotRecursive); } /** * Entry point shared between the new CLI `nix daemon` and old CLI * `nix-daemon`. + * + * @param forceTrustClientOpt See `daemonLoop()` and the parameter with + * the same name over there for details. */ -static void runDaemon(bool stdio) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt) { if (stdio) { auto store = openUncachedStore(); - if (auto remoteStore = store.dynamic_pointer_cast()) + // If --force-untrusted is passed, we cannot forward the connection and + // must process it ourselves (before delegating to the next store) to + // force untrusting the client. + if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) forwardStdioConnection(*remoteStore); else - processStdioConnection(store); + // `Trusted` is passed in the auto (no override case) because we + // cannot see who is on the other side of a plain pipe. Limiting + // access to those is explicitly not `nix-daemon`'s responsibility. + processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); } else - daemonLoop(); + daemonLoop(forceTrustClientOpt); } static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; + std::optional isTrustedOpt = std::nullopt; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -452,11 +474,20 @@ static int main_nix_daemon(int argc, char * * argv) printVersion("nix-daemon"); else if (*arg == "--stdio") stdio = true; - else return false; + else if (*arg == "--force-trusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = Trusted; + } else if (*arg == "--force-untrusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = NotTrusted; + } else if (*arg == "--default-trust") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = std::nullopt; + } else return false; return true; }); - runDaemon(stdio); + runDaemon(stdio, isTrustedOpt); return 0; } @@ -482,7 +513,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false); + runDaemon(false, std::nullopt); } }; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index c7af9c92c..d880bef0a 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -62,7 +62,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); - PathSet context; + NixStringContext context; if (apply) { auto vApply = state->allocValue(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a6c6b947f..3db655aeb 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -438,7 +438,7 @@ struct CmdFlakeCheck : FlakeCommand if (auto attr = v.attrs->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); if (!path.pathExists()) throw Error("template '%s' refers to a non-existent path '%s'", attrPath, path); diff --git a/src/nix/flake.md b/src/nix/flake.md index d70f34eeb..456fd0ea1 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -381,10 +381,12 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without - confirmation so long as `accept-flake-config` is not set in the global - configuration. + set of options is allowed to be set without confirmation so long as [`accept-flake-config`](@docroot@/command-ref/conf-file.md#conf-accept-flake-config) is not enabled in the global configuration: + - [`bash-prompt`](@docroot@/command-ref/conf-file.md#conf-bash-prompt) + - [`bash-prompt-prefix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-prefix) + - [`bash-prompt-suffix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-suffix) + - [`flake-registry`](@docroot@/command-ref/conf-file.md#conf-flake-registry) + - [`commit-lockfile-summary`](@docroot@/command-ref/conf-file.md#conf-commit-lockfile-summary) ## Flake inputs diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 039608d48..3b2e225f6 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -70,7 +70,13 @@ std::tuple prefetchFile( the store. */ if (expectedHash) { hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { + .hash = { + .method = ingestionMethod, + .hash = *expectedHash, + }, + .references = {}, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -121,7 +127,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 88d4a3ce5..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,12 +200,22 @@ struct ProfileManifest auto narHash = hashString(htSHA256, sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + "profile", + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; StringSource source(sink.s); store->addToStore(info, source); diff --git a/tests/build-remote-trustless-after.sh b/tests/build-remote-trustless-after.sh new file mode 100644 index 000000000..19f59e6ae --- /dev/null +++ b/tests/build-remote-trustless-after.sh @@ -0,0 +1,2 @@ +outPath=$(readlink -f $TEST_ROOT/result) +grep 'FOO BAR BAZ' ${remoteDir}/${outPath} diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh new file mode 100644 index 000000000..5e3d5ae07 --- /dev/null +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -0,0 +1,29 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# We first build a dependency of the derivation we eventually want to +# build. +nix-build build-hook.nix -A passthru.input2 \ + -o "$TEST_ROOT/input2" \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --option system-features bar + +# Now when we go to build that downstream derivation, Nix will fail +# because we cannot trustlessly build input-addressed derivations with +# `inputDrv` dependencies. + +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +expectStderr 1 source build-remote-trustless.sh \ + | grepQuiet "you are not privileged to build input-addressed derivations" diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/build-remote-trustless-should-pass-0.sh new file mode 100644 index 000000000..2a7ebd8c6 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-0.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-store +proto=ssh + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/build-remote-trustless-should-pass-1.sh new file mode 100644 index 000000000..516bdf092 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-1.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-daemon +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/build-remote-trustless-should-pass-3.sh new file mode 100644 index 000000000..40f81da5a --- /dev/null +++ b/tests/build-remote-trustless-should-pass-3.sh @@ -0,0 +1,14 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +# Remote doesn't trusts us, but this is fine because we are only +# building (fixed) CA derivations. +file=build-hook-ca-fixed.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless.sh b/tests/build-remote-trustless.sh new file mode 100644 index 000000000..9df44e0c5 --- /dev/null +++ b/tests/build-remote-trustless.sh @@ -0,0 +1,14 @@ +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +remoteDir=$TEST_ROOT/remote + +# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# more details. +nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/local \ + --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh index 3615177e9..c1480fd17 100644 --- a/tests/ca/derivation-json.sh +++ b/tests/ca/derivation-json.sh @@ -16,6 +16,9 @@ drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] +# But the JSON is rejected without the experimental feature +expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" + # Without --dry-run it is actually written drvPath4=$(nix derivation add < $TEST_HOME/foo.json) [[ "$drvPath4" = "$drvPath3" ]] diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index 73554da8c..607bf0a8e 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -23,20 +23,64 @@ source common.sh # # Medium case, the configuration effects --help # grep_both_ways store gc --help -expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' -nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' +# Test settings that are gated on experimental features; the setting is ignored +# with a warning if the experimental feature is not enabled. The order of the +# `setting = value` lines in the configuration should not matter. + +# 'flakes' experimental-feature is disabled before, ignore and warn +NIX_CONFIG=' + experimental-features = nix-command + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is disabled after, ignore and warn +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled before, process +NIX_CONFIG=' + experimental-features = nix-command flakes + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled after, process +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command flakes +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +function exit_code_both_ways { + expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null + nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + + # Also, the order should not matter + expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null + nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null +} + +exit_code_both_ways show-config --flake-registry 'https://no' # Double check these are stable -nix --experimental-features '' --help -nix --experimental-features '' doctor --help -nix --experimental-features '' repl --help -nix --experimental-features '' upgrade-nix --help +nix --experimental-features '' --help 1>/dev/null +nix --experimental-features '' doctor --help 1>/dev/null +nix --experimental-features '' repl --help 1>/dev/null +nix --experimental-features '' upgrade-nix --help 1>/dev/null # These 3 arguments are currently given to all commands, which is wrong (as not # all care). To deal with fixing later, we simply make them require the # nix-command experimental features --- it so happens that the commands we wish # stabilizing to do not need them anyways. for arg in '--print-build-logs' '--offline' '--refresh'; do - nix --experimental-features 'nix-command' "$arg" --help - ! nix --experimental-features '' "$arg" --help + nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null + expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null done diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index c7dadf397..39d053a04 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -10,6 +10,15 @@ clearStore # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter +# `nix derivation add` with impure derivations work +drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) +nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json +drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +[[ "$drvPath" = "$drvPath2" ]] + +# But only with the experimental feature! +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" + nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) path1=$(echo $json | jq -r .[].outputs.out) diff --git a/tests/local.mk b/tests/local.mk index 6cb466e8e..7c3b42599 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -70,6 +70,10 @@ nix_tests = \ check-reqs.sh \ build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ + build-remote-trustless-should-pass-0.sh \ + build-remote-trustless-should-pass-1.sh \ + build-remote-trustless-should-pass-3.sh \ + build-remote-trustless-should-fail-0.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ diff --git a/tests/nix-daemon-untrusting.sh b/tests/nix-daemon-untrusting.sh new file mode 100755 index 000000000..bcdb70989 --- /dev/null +++ b/tests/nix-daemon-untrusting.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec nix-daemon --force-untrusted "$@" diff --git a/tests/recursive.nix b/tests/recursive.nix new file mode 100644 index 000000000..fa8cc04db --- /dev/null +++ b/tests/recursive.nix @@ -0,0 +1,56 @@ +with import ./config.nix; + +mkDerivation rec { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + mkdir $out + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts store add-path ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] + + # Build a derivation. + nix $opts build -L --impure --expr ' + with import ${./config.nix}; + mkDerivation { + name = "inner1"; + buildCommand = "echo $fnord blaat > $out"; + fnord = builtins.toFile "fnord" "fnord"; + } + ' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + ''; +} diff --git a/tests/recursive.sh b/tests/recursive.sh index 6335d44a5..638f06f85 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -12,63 +12,7 @@ rm -f $TEST_ROOT/result export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' - with import ./config.nix; - mkDerivation rec { - name = "recursive"; - dummy = builtins.toFile "dummy" "bla bla"; - SHELL = shell; - - # Note: this is a string without context. - unreachable = builtins.getEnv "unreachable"; - - NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; - - requiredSystemFeatures = [ "recursive-nix" ]; - - buildCommand = '\'\'' - mkdir $out - opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" - - PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH - - # Check that we can query/build paths in our input closure. - nix $opts path-info $dummy - nix $opts build $dummy - - # Make sure we cannot query/build paths not in out input closure. - [[ -e $unreachable ]] - (! nix $opts path-info $unreachable) - (! nix $opts build $unreachable) - - # Add something to the store. - echo foobar > foobar - foobar=$(nix $opts store add-path ./foobar) - - nix $opts path-info $foobar - nix $opts build $foobar - - # Add it to our closure. - ln -s $foobar $out/foobar - - [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] - - # Build a derivation. - nix $opts build -L --impure --expr '\'' - with import ${./config.nix}; - mkDerivation { - name = "inner1"; - buildCommand = "echo $fnord blaat > $out"; - fnord = builtins.toFile "fnord" "fnord"; - } - '\'' - - [[ $(nix $opts path-info --json ./result) =~ fnord ]] - - ln -s $(nix $opts path-info ./result) $out/inner1 - '\'\''; - } -' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]]