From 141cb9a706f73725fd4f8d62569f45bbbf9b6c3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 17:56:28 +0000 Subject: [PATCH] Make regular `copyPaths` only copy again The is new function parameter so just the build hook can opt into the remote-side building. --- src/build-remote/build-remote.cc | 7 +++-- src/libstore/store-api.cc | 53 ++++++++++++++++++++------------ src/libstore/store-api.hh | 20 ++++++++++-- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 402754c3c..ef943ee11 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -270,9 +270,11 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + auto copyStorePathImpl = sshStore2->isTrusting ? copyStorePathAdapter : copyOrBuildStorePath; + { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, sshStore2, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); + copyPaths(store, sshStore2, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute, copyStorePathImpl); } uploadLock = -1; @@ -285,7 +287,7 @@ connected: if (!result.success()) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { - copyPaths(store, sshStore2, {*drvPath}, NoRepair, NoCheckSigs, substitute); + copyPaths(store, sshStore2, {*drvPath}, NoRepair, NoCheckSigs, substitute, copyStorePathImpl); sshStore2->buildPaths({{*drvPath}}); } @@ -298,6 +300,7 @@ connected: Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); for (auto & i : missing) store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ + /* No `copyStorePathImpl` because we always trust ourselves. */ copyPaths(sshStore2, store, missing, NoRepair, NoCheckSigs, NoSubstitute); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cce95e4cf..ad773bb0f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -819,8 +819,40 @@ void copyStorePath(ref srcStore, ref dstStore, } +void copyStorePathAdapter(ref srcStore, ref dstStore, + const ValidPathInfo & info, RepairFlag repair, CheckSigsFlag checkSigs) +{ + copyStorePath(srcStore, dstStore, info.path, repair, checkSigs); +} + +void copyOrBuildStorePath(ref srcStore, ref dstStore, + const ValidPathInfo & info, RepairFlag repair, CheckSigsFlag checkSigs) +{ + auto storePath = info.path; + if (dstStore->isTrusting || info.ca) { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } else if (info.deriver && dstStore->storeDir == srcStore->storeDir) { + auto drvPath = *info.deriver; + auto outputMap = srcStore->queryDerivationOutputMap(drvPath); + auto p = std::find_if(outputMap.begin(), outputMap.end(), [&](auto & i) { + return i.second == storePath; + }); + // drv file is always CA + srcStore->ensurePath(drvPath); + copyStorePath(srcStore, dstStore, drvPath, repair, checkSigs); + dstStore->buildPaths({{ + drvPath, + p != outputMap.end() ? StringSet { p->first } : StringSet {}, + }}); + } else { + dstStore->ensurePath(storePath); + } +} + + std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute, + std::function, ref, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl) { auto valid = dstStore->queryValidPaths(storePaths, substitute); @@ -893,24 +925,7 @@ std::map copyPaths(ref srcStore, ref dstStor MaintainCount mc(nrRunning); showProgress(); try { - if (dstStore->isTrusting || info->ca) { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } else if (info->deriver && dstStore->storeDir == srcStore->storeDir) { - auto drvPath = *info->deriver; - auto outputMap = srcStore->queryDerivationOutputMap(drvPath); - auto p = std::find_if(outputMap.begin(), outputMap.end(), [&](auto & i) { - return i.second == storePath; - }); - // drv file is always CA - srcStore->ensurePath(drvPath); - copyStorePath(srcStore, dstStore, drvPath, repair, checkSigs); - dstStore->buildPaths({{ - drvPath, - p != outputMap.end() ? StringSet { p->first } : StringSet {}, - }}); - } else { - dstStore->ensurePath(storePath); - } + copyStorePathImpl(srcStore, dstStore, *info, repair, checkSigs); } catch (Error &e) { nrFailed++; if (!settings.keepGoing) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 696a2d3fd..5597ebb22 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -738,17 +738,33 @@ void copyStorePath(ref srcStore, ref dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); +/* copyStorePath wrapped to be used with `copyPaths`. */ +void copyStorePathAdapter(ref srcStore, ref dstStore, + const ValidPathInfo & info, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + +/* The more liberal alternative to `copyStorePathAdapter`, useful for remote + stores that do not trust us. */ +void copyOrBuildStorePath(ref srcStore, ref dstStore, + const ValidPathInfo & info, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + /* Copy store paths from one store to another. The paths may be copied in parallel. They are copied in a topologically sorted order (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for that. Returns a map of what each path was copied to the dstStore - as. */ + as. + + The `copyStorePathImpl` parameter allows doing something other than just + copying. For example, this is used with the build hook to allow the other + side to build dependencies we don't have permission to copy. This behavior + isn't just the default that way `nix copy` etc. still can be relied upon to + not build anything. */ std::map copyPaths(ref srcStore, ref dstStore, const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - SubstituteFlag substitute = NoSubstitute); + SubstituteFlag substitute = NoSubstitute, + std::function, ref, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl = copyStorePathAdapter); /* Copy the closure of the specified paths from one store to another. */