mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-13 01:36:15 +02:00
Merge pull request #9589 from obsidiansystems/floating-content-addressing-derivations-eval-store
Fix building CA derivations with and eval store
This commit is contained in:
commit
dfc0cee702
7 changed files with 76 additions and 20 deletions
|
@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation()
|
||||||
things being garbage collected while we're busy. */
|
things being garbage collected while we're busy. */
|
||||||
worker.evalStore.addTempRoot(drvPath);
|
worker.evalStore.addTempRoot(drvPath);
|
||||||
|
|
||||||
assert(worker.evalStore.isValidPath(drvPath));
|
/* Get the derivation. It is probably in the eval store, but it might be inthe main store:
|
||||||
|
|
||||||
/* Get the derivation. */
|
- Resolved derivation are resolved against main store realisations, and so must be stored there.
|
||||||
drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
|
|
||||||
|
- Dynamic derivations are built, and so are found in the main store.
|
||||||
|
*/
|
||||||
|
for (auto * drvStore : { &worker.evalStore, &worker.store }) {
|
||||||
|
if (drvStore->isValidPath(drvPath)) {
|
||||||
|
drv = std::make_unique<Derivation>(drvStore->readDerivation(drvPath));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert(drv);
|
||||||
|
|
||||||
haveDerivation();
|
haveDerivation();
|
||||||
}
|
}
|
||||||
|
@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution()
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Copy the input sources from the eval store to the build
|
/* Copy the input sources from the eval store to the build
|
||||||
store. */
|
store.
|
||||||
|
|
||||||
|
Note that some inputs might not be in the eval store because they
|
||||||
|
are (resolved) derivation outputs in a resolved derivation. */
|
||||||
if (&worker.evalStore != &worker.store) {
|
if (&worker.evalStore != &worker.store) {
|
||||||
RealisedPath::Set inputSrcs;
|
RealisedPath::Set inputSrcs;
|
||||||
for (auto & i : drv->inputSrcs)
|
for (auto & i : drv->inputSrcs)
|
||||||
inputSrcs.insert(i);
|
if (worker.evalStore.isValidPath(i))
|
||||||
|
inputSrcs.insert(i);
|
||||||
copyClosure(worker.evalStore, worker.store, inputSrcs);
|
copyClosure(worker.evalStore, worker.store, inputSrcs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -453,7 +466,7 @@ void DerivationGoal::repairClosure()
|
||||||
std::map<StorePath, StorePath> outputsToDrv;
|
std::map<StorePath, StorePath> outputsToDrv;
|
||||||
for (auto & i : inputClosure)
|
for (auto & i : inputClosure)
|
||||||
if (i.isDerivation()) {
|
if (i.isDerivation()) {
|
||||||
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i);
|
auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore);
|
||||||
for (auto & j : depOutputs)
|
for (auto & j : depOutputs)
|
||||||
if (j.second)
|
if (j.second)
|
||||||
outputsToDrv.insert_or_assign(*j.second, i);
|
outputsToDrv.insert_or_assign(*j.second, i);
|
||||||
|
@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised()
|
||||||
return *outPath;
|
return *outPath;
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
|
auto outMap = [&]{
|
||||||
|
for (auto * drvStore : { &worker.evalStore, &worker.store })
|
||||||
|
if (drvStore->isValidPath(depDrvPath))
|
||||||
|
return worker.store.queryDerivationOutputMap(depDrvPath, drvStore);
|
||||||
|
assert(false);
|
||||||
|
}();
|
||||||
|
|
||||||
auto outMapPath = outMap.find(outputName);
|
auto outMapPath = outMap.find(outputName);
|
||||||
if (outMapPath == outMap.end()) {
|
if (outMapPath == outMap.end()) {
|
||||||
throw Error(
|
throw Error(
|
||||||
|
@ -1085,8 +1104,12 @@ void DerivationGoal::resolvedFinished()
|
||||||
auto newRealisation = realisation;
|
auto newRealisation = realisation;
|
||||||
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
|
newRealisation.id = DrvOutput { initialOutput->outputHash, outputName };
|
||||||
newRealisation.signatures.clear();
|
newRealisation.signatures.clear();
|
||||||
if (!drv->type().isFixed())
|
if (!drv->type().isFixed()) {
|
||||||
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
|
auto & drvStore = worker.evalStore.isValidPath(drvPath)
|
||||||
|
? worker.evalStore
|
||||||
|
: worker.store;
|
||||||
|
newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
|
||||||
|
}
|
||||||
signRealisation(newRealisation);
|
signRealisation(newRealisation);
|
||||||
worker.store.registerDrvOutput(newRealisation);
|
worker.store.registerDrvOutput(newRealisation);
|
||||||
}
|
}
|
||||||
|
@ -1379,7 +1402,10 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
|
||||||
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
|
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
|
||||||
return res;
|
return res;
|
||||||
} else {
|
} else {
|
||||||
return worker.store.queryPartialDerivationOutputMap(drvPath);
|
for (auto * drvStore : { &worker.evalStore, &worker.store })
|
||||||
|
if (drvStore->isValidPath(drvPath))
|
||||||
|
return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore);
|
||||||
|
assert(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1392,7 +1418,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
|
||||||
res.insert_or_assign(name, *output.second);
|
res.insert_or_assign(name, *output.second);
|
||||||
return res;
|
return res;
|
||||||
} else {
|
} else {
|
||||||
return worker.store.queryDerivationOutputMap(drvPath);
|
for (auto * drvStore : { &worker.evalStore, &worker.store })
|
||||||
|
if (drvStore->isValidPath(drvPath))
|
||||||
|
return worker.store.queryDerivationOutputMap(drvPath, drvStore);
|
||||||
|
assert(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -331,8 +331,11 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
std::map<DrvOutput, StorePath> drvOutputReferences(
|
std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
Store & store,
|
Store & store,
|
||||||
const Derivation & drv,
|
const Derivation & drv,
|
||||||
const StorePath & outputPath)
|
const StorePath & outputPath,
|
||||||
|
Store * evalStore_)
|
||||||
{
|
{
|
||||||
|
auto & evalStore = evalStore_ ? *evalStore_ : store;
|
||||||
|
|
||||||
std::set<Realisation> inputRealisations;
|
std::set<Realisation> inputRealisations;
|
||||||
|
|
||||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
|
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumRealisations;
|
||||||
|
@ -340,7 +343,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||||
if (!inputNode.value.empty()) {
|
if (!inputNode.value.empty()) {
|
||||||
auto outputHashes =
|
auto outputHashes =
|
||||||
staticOutputHashes(store, store.readDerivation(inputDrv));
|
staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
|
||||||
for (const auto & outputName : inputNode.value) {
|
for (const auto & outputName : inputNode.value) {
|
||||||
auto outputHash = get(outputHashes, outputName);
|
auto outputHash = get(outputHashes, outputName);
|
||||||
if (!outputHash)
|
if (!outputHash)
|
||||||
|
@ -362,7 +365,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
|
SingleDerivedPath next = SingleDerivedPath::Built { d, outputName };
|
||||||
accumRealisations(
|
accumRealisations(
|
||||||
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
||||||
resolveDerivedPath(store, next),
|
resolveDerivedPath(store, next, evalStore_),
|
||||||
childNode);
|
childNode);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -943,6 +943,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||||
std::map<DrvOutput, StorePath> drvOutputReferences(
|
std::map<DrvOutput, StorePath> drvOutputReferences(
|
||||||
Store & store,
|
Store & store,
|
||||||
const Derivation & drv,
|
const Derivation & drv,
|
||||||
const StorePath & outputPath);
|
const StorePath & outputPath,
|
||||||
|
Store * evalStore = nullptr);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -462,7 +462,7 @@ static void main_nix_build(int argc, char * * argv)
|
||||||
if (dryRun) return;
|
if (dryRun) return;
|
||||||
|
|
||||||
if (shellDrv) {
|
if (shellDrv) {
|
||||||
auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value());
|
auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore);
|
||||||
shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
|
shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -515,7 +515,7 @@ static void main_nix_build(int argc, char * * argv)
|
||||||
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
|
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputClosure;
|
||||||
|
|
||||||
accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
|
||||||
auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv);
|
auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore);
|
||||||
for (auto & i : inputNode.value) {
|
for (auto & i : inputNode.value) {
|
||||||
auto o = outputs.at(i);
|
auto o = outputs.at(i);
|
||||||
store->computeFSClosure(*o, inputs);
|
store->computeFSClosure(*o, inputs);
|
||||||
|
@ -653,7 +653,7 @@ static void main_nix_build(int argc, char * * argv)
|
||||||
if (counter)
|
if (counter)
|
||||||
drvPrefix += fmt("-%d", counter + 1);
|
drvPrefix += fmt("-%d", counter + 1);
|
||||||
|
|
||||||
auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath);
|
auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore);
|
||||||
|
|
||||||
auto maybeOutputPath = builtOutputs.at(outputName);
|
auto maybeOutputPath = builtOutputs.at(outputName);
|
||||||
assert(maybeOutputPath);
|
assert(maybeOutputPath);
|
||||||
|
|
10
tests/functional/ca/eval-store.sh
Normal file
10
tests/functional/ca/eval-store.sh
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# Ensure that garbage collection works properly with ca derivations
|
||||||
|
|
||||||
|
source common.sh
|
||||||
|
|
||||||
|
export NIX_TESTS_CA_BY_DEFAULT=1
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
source eval-store.sh
|
|
@ -5,6 +5,7 @@ ca-tests := \
|
||||||
$(d)/concurrent-builds.sh \
|
$(d)/concurrent-builds.sh \
|
||||||
$(d)/derivation-json.sh \
|
$(d)/derivation-json.sh \
|
||||||
$(d)/duplicate-realisation-in-closure.sh \
|
$(d)/duplicate-realisation-in-closure.sh \
|
||||||
|
$(d)/eval-store.sh \
|
||||||
$(d)/gc.sh \
|
$(d)/gc.sh \
|
||||||
$(d)/import-derivation.sh \
|
$(d)/import-derivation.sh \
|
||||||
$(d)/new-build-cmd.sh \
|
$(d)/new-build-cmd.sh \
|
||||||
|
|
|
@ -11,7 +11,16 @@ rm -rf "$eval_store"
|
||||||
|
|
||||||
nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
|
nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
|
||||||
[[ -e $TEST_ROOT/result/foobar ]]
|
[[ -e $TEST_ROOT/result/foobar ]]
|
||||||
(! ls $NIX_STORE_DIR/*.drv)
|
if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
|
||||||
|
# Resolved CA derivations are written to store for building
|
||||||
|
#
|
||||||
|
# TODO when we something more systematic
|
||||||
|
# (https://github.com/NixOS/nix/issues/5025) that distinguishes
|
||||||
|
# between scratch storage for building and the final destination
|
||||||
|
# store, we'll be able to make this unconditional again -- resolved
|
||||||
|
# derivations should only appear in the scratch store.
|
||||||
|
(! ls $NIX_STORE_DIR/*.drv)
|
||||||
|
fi
|
||||||
ls $eval_store/nix/store/*.drv
|
ls $eval_store/nix/store/*.drv
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
|
@ -26,5 +35,8 @@ rm -rf "$eval_store"
|
||||||
|
|
||||||
nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
|
nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result"
|
||||||
[[ -e $TEST_ROOT/result/foobar ]]
|
[[ -e $TEST_ROOT/result/foobar ]]
|
||||||
(! ls $NIX_STORE_DIR/*.drv)
|
if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
|
||||||
|
# See above
|
||||||
|
(! ls $NIX_STORE_DIR/*.drv)
|
||||||
|
fi
|
||||||
ls $eval_store/nix/store/*.drv
|
ls $eval_store/nix/store/*.drv
|
||||||
|
|
Loading…
Reference in a new issue