Allow substituting from different storeDir

Substituters can substitute from one store dir to another with a
little bit of help. The store api just needs to have a CA so it can
recompute the store path based on the new store dir. We can only do
this for fixed output derivations with no references, though.

src/libstore/local-store.cc

@@ -855,16 +855,24 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
 
 
 void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
 {
     if (!settings.useSubstitutes) return;
     for (auto & sub : getDefaultSubstituters()) {
-        if (sub->storeDir != storeDir) continue;
+        for (auto & path_ : paths) {
-        for (auto & path : paths) {
+            auto path(path_.clone());
-            if (infos.count(path)) continue;
             debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
             try {
                 auto info = sub->queryPathInfo(path);
+
+                auto ca = pathsCA.find(printStorePath(path));
+                if (sub->storeDir != storeDir && info->references.empty() && ca != pathsCA.end()) {
+                    if (!hasPrefix(ca->second, "fixed:"))
+                        continue;
+                    // recompute store path so that we can use a fixed output ca
+                    path = sub->makeStorePath("output:out", hashString(htSHA256, ca->second), path.name());
+                } else continue;
+
                 auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
                     std::shared_ptr<const ValidPathInfo>(info));
                 infos.insert_or_assign(path.clone(), SubstitutablePathInfo{

src/libstore/local-store.hh

@@ -142,7 +142,8 @@ public:
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
     void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) override;
+        SubstitutablePathInfos & infos,
+        std::map<std::string, std::string> pathsCA = {}) override;
 
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs,

src/libstore/misc.cc

@@ -108,6 +108,27 @@ void Store::computeFSClosure(const StorePath & startPath,
 }
 
 
+static std::optional<std::string> getDerivationCA(ref<Derivation> drv)
+{
+    auto outputHashMode = drv->env.find("outputHashMode");
+    auto outputHashAlgo = drv->env.find("outputHashAlgo");
+    auto outputHash = drv->env.find("outputHash");
+    if (outputHashMode != drv->env.end() && outputHashAlgo != drv->env.end() && outputHash != drv->env.end()) {
+        auto ht = parseHashType(outputHashAlgo->second);
+        auto h = Hash(outputHash->second, ht);
+        FileIngestionMethod ingestionMethod;
+        if (outputHashMode->second == "recursive")
+            ingestionMethod = FileIngestionMethod::Recursive;
+        else if (outputHashMode->second == "flat")
+            ingestionMethod = FileIngestionMethod::Flat;
+        else
+            throw Error("unknown ingestion method: '%s'", outputHashMode->second);
+        return makeFixedOutputCA(ingestionMethod, h);
+    }
+
+    return std::nullopt;
+}
+
 void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
     StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
     unsigned long long & downloadSize_, unsigned long long & narSize_)
@@ -159,7 +180,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
         SubstitutablePathInfos infos;
         StorePathSet paths; // FIXME
         paths.insert(outPath.clone());
-        querySubstitutablePathInfos(paths, infos);
+
+        std::map<std::string, std::string> pathsCA = {};
+        if (auto ca = getDerivationCA(drv))
+            pathsCA.insert({outPathS, *ca});
+        querySubstitutablePathInfos(paths, infos, pathsCA);
 
         if (infos.empty()) {
             drvState_->lock()->done = true;

src/libstore/remote-store.cc

@@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
 
 
 void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
-    SubstitutablePathInfos & infos)
+    SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA)
 {
     if (paths.empty()) return;
 

src/libstore/remote-store.hh

@@ -58,7 +58,7 @@ public:
     StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
 
     void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) override;
+        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) override;
 
     void addToStore(const ValidPathInfo & info, Source & nar,
         RepairFlag repair, CheckSigsFlag checkSigs,

src/libstore/store-api.cc

@@ -580,6 +580,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
 
     uint64_t total = 0;
 
+    // recompute store path on the chance dstStore does it differently
+    if (hasPrefix(info->ca, "fixed:") && info->references.empty()) {
+        auto info2 = make_ref<ValidPathInfo>(*info);
+        info2->path = dstStore->makeStorePath("output:out", hashString(htSHA256, info->ca), storePath.name());
+        info = info2;
+    }
+
     if (!info->narHash) {
         StringSink sink;
         srcStore->narFromPath({storePath}, sink);

src/libstore/store-api.hh

@@ -445,7 +445,7 @@ public:
        sizes) of a set of paths.  If a path does not have substitute
        info, it's omitted from the resulting ‘infos’ map. */
     virtual void querySubstitutablePathInfos(const StorePathSet & paths,
-        SubstitutablePathInfos & infos) { return; };
+        SubstitutablePathInfos & infos, std::map<std::string, std::string> pathsCA = {}) { return; };
 
     /* Import a path into the store. */
     virtual void addToStore(const ValidPathInfo & info, Source & narSource,