SourceAccessor::readFile(): Support reading into a sink

src/libfetchers/fs-input-accessor.cc

@@ -19,11 +19,14 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
     {
     }
 
-    std::string readFile(const CanonPath & path) override
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override
     {
         auto absPath = makeAbsPath(path);
         checkAllowed(absPath);
-        return PosixSourceAccessor::readFile(absPath);
+        PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
     }
 
     bool pathExists(const CanonPath & path) override

src/libfetchers/input-accessor.cc

@@ -17,7 +17,7 @@ StorePath InputAccessor::fetchToStore(
         if (method == FileIngestionMethod::Recursive)
             dumpPath(path, sink, filter ? *filter : defaultPathFilter);
         else
-            sink(readFile(path)); // FIXME: stream
+            readFile(path, sink);
     });
 
     auto storePath =

src/libutil/archive.cc

@@ -44,12 +44,15 @@ void SourceAccessor::dumpPath(
 {
     auto dumpContents = [&](const CanonPath & path)
     {
-        /* It would be nice if this was streaming, but we need the
-           size before the contents. */
-        auto s = readFile(path);
-        sink << "contents" << s.size();
-        sink(s);
-        writePadding(s.size(), sink);
+        sink << "contents";
+        std::optional<uint64_t> size;
+        readFile(path, sink, [&](uint64_t _size)
+        {
+            size = _size;
+            sink << _size;
+        });
+        assert(size);
+        writePadding(*size, sink);
     };
 
     std::function<void(const CanonPath & path)> dump;

src/libutil/source-accessor.cc

@@ -10,6 +10,28 @@ SourceAccessor::SourceAccessor()
 {
 }
 
+std::string SourceAccessor::readFile(const CanonPath & path)
+{
+    StringSink sink;
+    std::optional<uint64_t> size;
+    readFile(path, sink, [&](uint64_t _size)
+    {
+        size = _size;
+    });
+    assert(size && *size == sink.s.size());
+    return std::move(sink.s);
+}
+
+void SourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
+{
+    auto s = readFile(path);
+    sizeCallback(s.size());
+    sink(s);
+}
+
 Hash SourceAccessor::hashPath(
     const CanonPath & path,
     PathFilter & filter,
@@ -33,9 +55,41 @@ std::string SourceAccessor::showPath(const CanonPath & path)
     return path.abs();
 }
 
-std::string PosixSourceAccessor::readFile(const CanonPath & path)
+void PosixSourceAccessor::readFile(
+    const CanonPath & path,
+    Sink & sink,
+    std::function<void(uint64_t)> sizeCallback)
 {
-    return nix::readFile(path.abs());
+    // FIXME: add O_NOFOLLOW since symlinks should be resolved by the
+    // caller?
+    AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+
+    struct stat st;
+    if (fstat(fd.get(), &st) == -1)
+        throw SysError("statting file");
+
+    sizeCallback(st.st_size);
+
+    off_t left = st.st_size;
+
+    std::vector<unsigned char> buf(64 * 1024);
+    while (left) {
+        checkInterrupt();
+        ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));
+        if (rd == -1) {
+            if (errno != EINTR)
+                throw SysError("reading from file '%s'", showPath(path));
+        }
+        else if (rd == 0)
+            throw SysError("unexpected end-of-file reading '%s'", showPath(path));
+        else {
+            assert(rd <= left);
+            sink({(char *) buf.data(), (size_t) rd});
+            left -= rd;
+        }
+    }
 }
 
 bool PosixSourceAccessor::pathExists(const CanonPath & path)

src/libutil/source-accessor.hh

@@ -5,6 +5,8 @@
 
 namespace nix {
 
+struct Sink;
+
 /**
  * A read-only filesystem abstraction. This is used by the Nix
  * evaluator and elsewhere for accessing sources in various
@@ -20,7 +22,23 @@ struct SourceAccessor
     virtual ~SourceAccessor()
     { }
 
-    virtual std::string readFile(const CanonPath & path) = 0;
+    /**
+     * Return the contents of a file as a string.
+     */
+    virtual std::string readFile(const CanonPath & path);
+
+    /**
+     * Write the contents of a file as a sink. `sizeCallback` must be
+     * called with the size of the file before any data is written to
+     * the sink.
+     *
+     * Note: subclasses of `SourceAccessor` need to implement at least
+     * one of the `readFile()` variants.
+     */
+    virtual void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
 
     virtual bool pathExists(const CanonPath & path) = 0;
 
@@ -97,7 +115,10 @@ struct PosixSourceAccessor : SourceAccessor
      */
     time_t mtime = 0;
 
-    std::string readFile(const CanonPath & path) override;
+    void readFile(
+        const CanonPath & path,
+        Sink & sink,
+        std::function<void(uint64_t)> sizeCallback) override;
 
     bool pathExists(const CanonPath & path) override;