Merge pull request #9265 from obsidiansystems/better-parse-sink

Make `ParseSink` a bit better
This commit is contained in:
John Ericson 2023-11-02 09:28:29 -04:00 committed by GitHub
commit 90de958637
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 132 additions and 55 deletions

View file

@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
eagerly consume the entire stream it's given, past the
length of the Nar. */
TeeSource savedNARSource(from, saved);
ParseSink sink; /* null sink; just parse the NAR */
NullParseSink sink; /* just parse the NAR */
parseDump(sink, savedNARSource);
} else {
/* Incrementally parse the NAR file, stripping the
metadata, and streaming the sole file we expect into
`saved`. */
RetrieveRegularNARSink savedRegular { saved };
RegularFileSink savedRegular { saved };
parseDump(savedRegular, from);
if (!savedRegular.regular) throw Error("regular file expected");
}
@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
source = std::make_unique<TunnelSource>(from, to);
else {
TeeSource tee { from, saved };
ParseSink ether;
NullParseSink ether;
parseDump(ether, tee);
source = std::make_unique<StringSource>(saved.s);
}

View file

@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
/* Extract the NAR from the source. */
StringSink saved;
TeeSource tee { source, saved };
ParseSink ether;
NullParseSink ether;
parseDump(ether, tee);
uint32_t magic = readInt(source);

View file

@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
bool narRead = false;
Finally cleanup = [&]() {
if (!narRead) {
ParseSink sink;
NullParseSink sink;
parseDump(sink, source);
}
};

View file

@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
/* Note that fileSink and unusualHashTee must be mutually exclusive, since
they both write to caHashSink. Note that that requisite is currently true
because the former is only used in the flat case. */
RetrieveRegularNARSink fileSink { caHashSink };
RegularFileSink fileSink { caHashSink };
TeeSink unusualHashTee { narHashSink, caHashSink };
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
information to narSink. */
TeeSource tapped { *fileSource, narSink };
ParseSink blank;
NullParseSink blank;
auto & parseSink = method == FileIngestionMethod::Flat
? fileSink
: blank;
? (ParseSink &) fileSink
: (ParseSink &) blank;
/* The information that flows from tapped (besides being replicated in
narSink), is now put in parseSink. */

View file

@ -5,12 +5,6 @@
#include <strings.h> // for strcasecmp
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
#include <fcntl.h>
#include "archive.hh"
#include "util.hh"
#include "config.hh"
@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink)
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
// we should just forward all data directly without parsing.
ParseSink parseSink; /* null sink; just parse the NAR */
NullParseSink parseSink; /* just parse the NAR */
TeeSource wrapper { source, sink };

View file

@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
*/
void dumpString(std::string_view s, Sink & sink);
/**
* If the NAR archive contains a single file at top-level, then save
* the contents of the file to `s`. Otherwise barf.
*/
struct RetrieveRegularNARSink : ParseSink
{
bool regular = true;
Sink & sink;
RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path) override
{
regular = false;
}
void receiveContents(std::string_view data) override
{
sink(data);
}
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}
};
void parseDump(ParseSink & sink, Source & source);
void restorePath(const Path & path, Source & source);

View file

@ -5,6 +5,54 @@
namespace nix {
void copyRecursive(
SourceAccessor & accessor, const CanonPath & from,
ParseSink & sink, const Path & to)
{
auto stat = accessor.lstat(from);
switch (stat.type) {
case SourceAccessor::tSymlink:
{
sink.createSymlink(to, accessor.readLink(from));
}
case SourceAccessor::tRegular:
{
sink.createRegularFile(to);
if (stat.isExecutable)
sink.isExecutable();
LambdaSink sink2 {
[&](auto d) {
sink.receiveContents(d);
}
};
accessor.readFile(from, sink2, [&](uint64_t size) {
sink.preallocateContents(size);
});
break;
}
case SourceAccessor::tDirectory:
{
sink.createDirectory(to);
for (auto & [name, _] : accessor.readDirectory(from)) {
copyRecursive(
accessor, from + name,
sink, to + "/" + name);
break;
}
}
case SourceAccessor::tMisc:
throw Error("file '%1%' has an unsupported type", from);
default:
abort();
}
}
struct RestoreSinkSettings : Config
{
Setting<bool> preallocateContents{this, false, "preallocate-contents",

View file

@ -3,6 +3,7 @@
#include "types.hh"
#include "serialise.hh"
#include "source-accessor.hh"
namespace nix {
@ -11,32 +12,93 @@ namespace nix {
*/
struct ParseSink
{
virtual void createDirectory(const Path & path) { };
virtual void createDirectory(const Path & path) = 0;
virtual void createRegularFile(const Path & path) { };
virtual void closeRegularFile() { };
virtual void isExecutable() { };
virtual void createRegularFile(const Path & path) = 0;
virtual void receiveContents(std::string_view data) = 0;
virtual void isExecutable() = 0;
virtual void closeRegularFile() = 0;
virtual void createSymlink(const Path & path, const std::string & target) = 0;
/**
* An optimization. By default, do nothing.
*/
virtual void preallocateContents(uint64_t size) { };
virtual void receiveContents(std::string_view data) { };
virtual void createSymlink(const Path & path, const std::string & target) { };
};
/**
* Recusively copy file system objects from the source into the sink.
*/
void copyRecursive(
SourceAccessor & accessor, const CanonPath & sourcePath,
ParseSink & sink, const Path & destPath);
/**
* Ignore everything and do nothing
*/
struct NullParseSink : ParseSink
{
void createDirectory(const Path & path) override { }
void receiveContents(std::string_view data) override { }
void createSymlink(const Path & path, const std::string & target) override { }
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
};
/**
* Write files at the given path
*/
struct RestoreSink : ParseSink
{
Path dstPath;
AutoCloseFD fd;
void createDirectory(const Path & path) override;
void createRegularFile(const Path & path) override;
void closeRegularFile() override;
void isExecutable() override;
void preallocateContents(uint64_t size) override;
void receiveContents(std::string_view data) override;
void isExecutable() override;
void closeRegularFile() override;
void createSymlink(const Path & path, const std::string & target) override;
void preallocateContents(uint64_t size) override;
private:
AutoCloseFD fd;
};
/**
* Restore a single file at the top level, passing along
* `receiveContents` to the underlying `Sink`. For anything but a single
* file, set `regular = true` so the caller can fail accordingly.
*/
struct RegularFileSink : ParseSink
{
bool regular = true;
Sink & sink;
RegularFileSink(Sink & sink) : sink(sink) { }
void createDirectory(const Path & path) override
{
regular = false;
}
void receiveContents(std::string_view data) override
{
sink(data);
}
void createSymlink(const Path & path, const std::string & target) override
{
regular = false;
}
void createRegularFile(const Path & path) override { }
void closeRegularFile() override { }
void isExecutable() override { }
};
}