Merge pull request #9265 from obsidiansystems/better-parse-sink

Make `ParseSink` a bit better

src/libstore/daemon.cc

@@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                        eagerly consume the entire stream it's given, past the
                        length of the Nar. */
                     TeeSource savedNARSource(from, saved);
-                    ParseSink sink; /* null sink; just parse the NAR */
+                    NullParseSink sink; /* just parse the NAR */
                     parseDump(sink, savedNARSource);
                 } else {
                     /* Incrementally parse the NAR file, stripping the
                        metadata, and streaming the sole file we expect into
                        `saved`. */
-                    RetrieveRegularNARSink savedRegular { saved };
+                    RegularFileSink savedRegular { saved };
                     parseDump(savedRegular, from);
                     if (!savedRegular.regular) throw Error("regular file expected");
                 }
@@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
                 source = std::make_unique<TunnelSource>(from, to);
             else {
                 TeeSource tee { from, saved };
-                ParseSink ether;
+                NullParseSink ether;
                 parseDump(ether, tee);
                 source = std::make_unique<StringSource>(saved.s);
             }

src/libstore/export-import.cc

@@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         /* Extract the NAR from the source. */
         StringSink saved;
         TeeSource tee { source, saved };
-        ParseSink ether;
+        NullParseSink ether;
         parseDump(ether, tee);
 
         uint32_t magic = readInt(source);

src/libstore/local-store.cc

@@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
     bool narRead = false;
     Finally cleanup = [&]() {
         if (!narRead) {
-            ParseSink sink;
+            NullParseSink sink;
             parseDump(sink, source);
         }
     };

src/libstore/store-api.cc

@@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
     /* Note that fileSink and unusualHashTee must be mutually exclusive, since
        they both write to caHashSink. Note that that requisite is currently true
        because the former is only used in the flat case. */
-    RetrieveRegularNARSink fileSink { caHashSink };
+    RegularFileSink fileSink { caHashSink };
     TeeSink unusualHashTee { narHashSink, caHashSink };
 
     auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
@@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
        information to narSink. */
     TeeSource tapped { *fileSource, narSink };
 
-    ParseSink blank;
+    NullParseSink blank;
     auto & parseSink = method == FileIngestionMethod::Flat
-        ? fileSink
-        : blank;
+        ? (ParseSink &) fileSink
+        : (ParseSink &) blank;
 
     /* The information that flows from tapped (besides being replicated in
        narSink), is now put in parseSink. */

src/libutil/archive.cc

@@ -5,12 +5,6 @@
 
 #include <strings.h> // for strcasecmp
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <dirent.h>
-#include <fcntl.h>
-
 #include "archive.hh"
 #include "util.hh"
 #include "config.hh"
@@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink)
     // FIXME: if 'source' is the output of dumpPath() followed by EOF,
     // we should just forward all data directly without parsing.
 
-    ParseSink parseSink; /* null sink; just parse the NAR */
+    NullParseSink parseSink; /* just parse the NAR */
 
     TeeSource wrapper { source, sink };
 

src/libutil/archive.hh

@@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
  */
 void dumpString(std::string_view s, Sink & sink);
 
-/**
- * If the NAR archive contains a single file at top-level, then save
- * the contents of the file to `s`.  Otherwise barf.
- */
-struct RetrieveRegularNARSink : ParseSink
-{
-    bool regular = true;
-    Sink & sink;
-
-    RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
-
-    void createDirectory(const Path & path) override
-    {
-        regular = false;
-    }
-
-    void receiveContents(std::string_view data) override
-    {
-        sink(data);
-    }
-
-    void createSymlink(const Path & path, const std::string & target) override
-    {
-        regular = false;
-    }
-};
-
 void parseDump(ParseSink & sink, Source & source);
 
 void restorePath(const Path & path, Source & source);

src/libutil/fs-sink.cc

@@ -5,6 +5,54 @@
 
 namespace nix {
 
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & from,
+    ParseSink & sink, const Path & to)
+{
+    auto stat = accessor.lstat(from);
+
+    switch (stat.type) {
+    case SourceAccessor::tSymlink:
+    {
+        sink.createSymlink(to, accessor.readLink(from));
+    }
+
+    case SourceAccessor::tRegular:
+    {
+        sink.createRegularFile(to);
+        if (stat.isExecutable)
+            sink.isExecutable();
+        LambdaSink sink2 {
+            [&](auto d) {
+                sink.receiveContents(d);
+            }
+        };
+        accessor.readFile(from, sink2, [&](uint64_t size) {
+            sink.preallocateContents(size);
+        });
+        break;
+    }
+
+    case SourceAccessor::tDirectory:
+    {
+        sink.createDirectory(to);
+        for (auto & [name, _] : accessor.readDirectory(from)) {
+            copyRecursive(
+                accessor, from + name,
+                sink, to + "/" + name);
+            break;
+        }
+    }
+
+    case SourceAccessor::tMisc:
+        throw Error("file '%1%' has an unsupported type", from);
+
+    default:
+        abort();
+    }
+}
+
+
 struct RestoreSinkSettings : Config
 {
     Setting<bool> preallocateContents{this, false, "preallocate-contents",

src/libutil/fs-sink.hh

@@ -3,6 +3,7 @@
 
 #include "types.hh"
 #include "serialise.hh"
+#include "source-accessor.hh"
 
 namespace nix {
 
@@ -11,32 +12,93 @@ namespace nix {
  */
 struct ParseSink
 {
-    virtual void createDirectory(const Path & path) { };
+    virtual void createDirectory(const Path & path) = 0;
 
-    virtual void createRegularFile(const Path & path) { };
-    virtual void closeRegularFile() { };
-    virtual void isExecutable() { };
+    virtual void createRegularFile(const Path & path) = 0;
+    virtual void receiveContents(std::string_view data) = 0;
+    virtual void isExecutable() = 0;
+    virtual void closeRegularFile() = 0;
+
+    virtual void createSymlink(const Path & path, const std::string & target) = 0;
+
+    /**
+     * An optimization. By default, do nothing.
+     */
     virtual void preallocateContents(uint64_t size) { };
-    virtual void receiveContents(std::string_view data) { };
-
-    virtual void createSymlink(const Path & path, const std::string & target) { };
 };
 
+/**
+ * Recusively copy file system objects from the source into the sink.
+ */
+void copyRecursive(
+    SourceAccessor & accessor, const CanonPath & sourcePath,
+    ParseSink & sink, const Path & destPath);
+
+/**
+ * Ignore everything and do nothing
+ */
+struct NullParseSink : ParseSink
+{
+    void createDirectory(const Path & path) override { }
+    void receiveContents(std::string_view data) override { }
+    void createSymlink(const Path & path, const std::string & target) override { }
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
+};
+
+/**
+ * Write files at the given path
+ */
 struct RestoreSink : ParseSink
 {
     Path dstPath;
-    AutoCloseFD fd;
-
 
     void createDirectory(const Path & path) override;
 
     void createRegularFile(const Path & path) override;
-    void closeRegularFile() override;
-    void isExecutable() override;
-    void preallocateContents(uint64_t size) override;
     void receiveContents(std::string_view data) override;
+    void isExecutable() override;
+    void closeRegularFile() override;
 
     void createSymlink(const Path & path, const std::string & target) override;
+
+    void preallocateContents(uint64_t size) override;
+
+private:
+    AutoCloseFD fd;
+};
+
+/**
+ * Restore a single file at the top level, passing along
+ * `receiveContents` to the underlying `Sink`. For anything but a single
+ * file, set `regular = true` so the caller can fail accordingly.
+ */
+struct RegularFileSink : ParseSink
+{
+    bool regular = true;
+    Sink & sink;
+
+    RegularFileSink(Sink & sink) : sink(sink) { }
+
+    void createDirectory(const Path & path) override
+    {
+        regular = false;
+    }
+
+    void receiveContents(std::string_view data) override
+    {
+        sink(data);
+    }
+
+    void createSymlink(const Path & path, const std::string & target) override
+    {
+        regular = false;
+    }
+
+    void createRegularFile(const Path & path) override { }
+    void closeRegularFile() override { }
+    void isExecutable() override { }
 };
 
 }