fetchClosure: Allow a path to be rewritten to CA on the fly

The advantage is that the resulting closure doesn't need to be signed,
so you don't need to configure any binary cache keys on the client.

src/libexpr/primops/fetchClosure.cc

@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "store-api.hh"
+#include "make-content-addressed.hh"
 
 namespace nix {
 
@@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
 
     std::optional<std::string> fromStoreUrl;
     std::optional<StorePath> fromPath;
+    bool toCA = false;
+    std::optional<StorePath> toPath;
 
     for (auto & attr : *args[0]->attrs) {
         if (attr.name == "fromPath") {
@@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
             fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
         }
 
+        else if (attr.name == "toPath") {
+            state.forceValue(*attr.value, *attr.pos);
+            toCA = true;
+            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+                PathSet context;
+                toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+            }
+        }
+
         else if (attr.name == "fromStore")
             fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
 
@@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
     // FIXME: only allow some "trusted" store types (like BinaryCacheStore).
     auto fromStore = openStore(*fromStoreUrl);
 
-    copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+    if (toCA) {
+        auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+        auto i = remappings.find(*fromPath);
+        assert(i != remappings.end());
+        if (toPath && *toPath != i->second)
+            throw Error({
+                .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                    state.store->printStorePath(*fromPath),
+                    state.store->printStorePath(i->second),
+                    state.store->printStorePath(*toPath)),
+                .errPos = pos
+            });
+        if (!toPath)
+            throw Error({
+                .msg = hintfmt(
+                    "rewriting '%s' to content-addressed form yielded '%s'; "
+                    "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                    state.store->printStorePath(*fromPath),
+                    state.store->printStorePath(i->second)),
+                .errPos = pos
+            });
+    } else {
+        copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+        toPath = fromPath;
+    }
 
     /* In pure mode, require a CA path. */
     if (evalSettings.pureEval) {
-        auto info = state.store->queryPathInfo(*fromPath);
+        auto info = state.store->queryPathInfo(*toPath);
         if (!info->isContentAddressed(*state.store))
             throw Error({
                 .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
-                    state.store->printStorePath(*fromPath)),
+                    state.store->printStorePath(*toPath)),
                 .errPos = pos
             });
     }
 
-    auto fromPathS = state.store->printStorePath(*fromPath);
+    auto toPathS = state.store->printStorePath(*toPath);
-    v.mkString(fromPathS, {fromPathS});
+    v.mkString(toPathS, {toPathS});
 }
 
 static RegisterPrimOp primop_fetchClosure({

src/libstore/make-content-addressed.cc

@@ -0,0 +1,79 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths)
+{
+    // FIXME: use closure of storePaths.
+
+    auto paths = srcStore.topoSortPaths(storePaths);
+
+    std::reverse(paths.begin(), paths.end());
+
+    std::map<StorePath, StorePath> remappings;
+
+    for (auto & path : paths) {
+        auto pathS = srcStore.printStorePath(path);
+        auto oldInfo = srcStore.queryPathInfo(path);
+        std::string oldHashPart(path.hashPart());
+
+        StringSink sink;
+        srcStore.narFromPath(path, sink);
+
+        StringMap rewrites;
+
+        StorePathSet references;
+        bool hasSelfReference = false;
+        for (auto & ref : oldInfo->references) {
+            if (ref == path)
+                hasSelfReference = true;
+            else {
+                auto i = remappings.find(ref);
+                auto replacement = i != remappings.end() ? i->second : ref;
+                // FIXME: warn about unremapped paths?
+                if (replacement != ref)
+                    rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+                references.insert(std::move(replacement));
+            }
+        }
+
+        sink.s = rewriteStrings(sink.s, rewrites);
+
+        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        hashModuloSink(sink.s);
+
+        auto narHash = hashModuloSink.finish().first;
+
+        ValidPathInfo info {
+            dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
+            narHash,
+        };
+        info.references = std::move(references);
+        if (hasSelfReference) info.references.insert(info.path);
+        info.narSize = sink.s.size();
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Recursive,
+            .hash = info.narHash,
+        };
+
+        printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path));
+
+        auto source = sinkToSource([&](Sink & nextSink) {
+            RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
+            rsink2(sink.s);
+            rsink2.flush();
+        });
+
+        dstStore.addToStore(info, *source);
+
+        remappings.insert_or_assign(std::move(path), std::move(info.path));
+    }
+
+    return remappings;
+}
+
+}

src/libstore/make-content-addressed.hh

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths);
+
+}

src/nix/make-content-addressable.cc

@@ -1,6 +1,6 @@
 #include "command.hh"
 #include "store-api.hh"
-#include "references.hh"
+#include "make-content-addressed.hh"
 #include "common-args.hh"
 #include "json.hh"
 
@@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
     void run(ref<Store> store, StorePaths && storePaths) override
     {
-        auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
+        auto remappings = makeContentAddressed(*store, *store,
+            StorePathSet(storePaths.begin(), storePaths.end()));
 
-        std::reverse(paths.begin(), paths.end());
+        if (json) {
-
+            JSONObject jsonRoot(std::cout);
-        std::map<StorePath, StorePath> remappings;
+            JSONObject jsonRewrites(jsonRoot.object("rewrites"));
-
+            for (auto & path : storePaths) {
-        auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
+                auto i = remappings.find(path);
-        auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
+                assert(i != remappings.end());
-
+                jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second));
-        for (auto & path : paths) {
+            }
-            auto pathS = store->printStorePath(path);
+        } else {
-            auto oldInfo = store->queryPathInfo(path);
+            for (auto & path : storePaths) {
-            std::string oldHashPart(path.hashPart());
+                auto i = remappings.find(path);
-
+                assert(i != remappings.end());
-            StringSink sink;
+                notice("rewrote '%s' to '%s'",
-            store->narFromPath(path, sink);
+                    store->printStorePath(path),
-
+                    store->printStorePath(i->second));
-            StringMap rewrites;
-
-            StorePathSet references;
-            bool hasSelfReference = false;
-            for (auto & ref : oldInfo->references) {
-                if (ref == path)
-                    hasSelfReference = true;
-                else {
-                    auto i = remappings.find(ref);
-                    auto replacement = i != remappings.end() ? i->second : ref;
-                    // FIXME: warn about unremapped paths?
-                    if (replacement != ref)
-                        rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
-                    references.insert(std::move(replacement));
-                }
             }
-
-            sink.s = rewriteStrings(sink.s, rewrites);
-
-            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-            hashModuloSink(sink.s);
-
-            auto narHash = hashModuloSink.finish().first;
-
-            ValidPathInfo info {
-                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
-                narHash,
-            };
-            info.references = std::move(references);
-            if (hasSelfReference) info.references.insert(info.path);
-            info.narSize = sink.s.size();
-            info.ca = FixedOutputHash {
-                .method = FileIngestionMethod::Recursive,
-                .hash = info.narHash,
-            };
-
-            if (!json)
-                notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
-            auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
-                rsink2(sink.s);
-                rsink2.flush();
-            });
-
-            store->addToStore(info, *source);
-
-            if (json)
-                jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
-            remappings.insert_or_assign(std::move(path), std::move(info.path));
         }
     }
 };