mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-22 05:56:15 +02:00
Merge pull request #9265 from obsidiansystems/better-parse-sink
Make `ParseSink` a bit better
This commit is contained in:
commit
90de958637
8 changed files with 132 additions and 55 deletions
|
@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
eagerly consume the entire stream it's given, past the
|
||||
length of the Nar. */
|
||||
TeeSource savedNARSource(from, saved);
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
NullParseSink sink; /* just parse the NAR */
|
||||
parseDump(sink, savedNARSource);
|
||||
} else {
|
||||
/* Incrementally parse the NAR file, stripping the
|
||||
metadata, and streaming the sole file we expect into
|
||||
`saved`. */
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
RegularFileSink savedRegular { saved };
|
||||
parseDump(savedRegular, from);
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
}
|
||||
|
@ -899,7 +899,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
|||
source = std::make_unique<TunnelSource>(from, to);
|
||||
else {
|
||||
TeeSource tee { from, saved };
|
||||
ParseSink ether;
|
||||
NullParseSink ether;
|
||||
parseDump(ether, tee);
|
||||
source = std::make_unique<StringSource>(saved.s);
|
||||
}
|
||||
|
|
|
@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
|
|||
/* Extract the NAR from the source. */
|
||||
StringSink saved;
|
||||
TeeSource tee { source, saved };
|
||||
ParseSink ether;
|
||||
NullParseSink ether;
|
||||
parseDump(ether, tee);
|
||||
|
||||
uint32_t magic = readInt(source);
|
||||
|
|
|
@ -1200,7 +1200,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
|||
bool narRead = false;
|
||||
Finally cleanup = [&]() {
|
||||
if (!narRead) {
|
||||
ParseSink sink;
|
||||
NullParseSink sink;
|
||||
parseDump(sink, source);
|
||||
}
|
||||
};
|
||||
|
|
|
@ -410,7 +410,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
|||
/* Note that fileSink and unusualHashTee must be mutually exclusive, since
|
||||
they both write to caHashSink. Note that that requisite is currently true
|
||||
because the former is only used in the flat case. */
|
||||
RetrieveRegularNARSink fileSink { caHashSink };
|
||||
RegularFileSink fileSink { caHashSink };
|
||||
TeeSink unusualHashTee { narHashSink, caHashSink };
|
||||
|
||||
auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
|
||||
|
@ -428,10 +428,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
|
|||
information to narSink. */
|
||||
TeeSource tapped { *fileSource, narSink };
|
||||
|
||||
ParseSink blank;
|
||||
NullParseSink blank;
|
||||
auto & parseSink = method == FileIngestionMethod::Flat
|
||||
? fileSink
|
||||
: blank;
|
||||
? (ParseSink &) fileSink
|
||||
: (ParseSink &) blank;
|
||||
|
||||
/* The information that flows from tapped (besides being replicated in
|
||||
narSink), is now put in parseSink. */
|
||||
|
|
|
@ -5,12 +5,6 @@
|
|||
|
||||
#include <strings.h> // for strcasecmp
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
|
||||
#include "archive.hh"
|
||||
#include "util.hh"
|
||||
#include "config.hh"
|
||||
|
@ -299,7 +293,7 @@ void copyNAR(Source & source, Sink & sink)
|
|||
// FIXME: if 'source' is the output of dumpPath() followed by EOF,
|
||||
// we should just forward all data directly without parsing.
|
||||
|
||||
ParseSink parseSink; /* null sink; just parse the NAR */
|
||||
NullParseSink parseSink; /* just parse the NAR */
|
||||
|
||||
TeeSource wrapper { source, sink };
|
||||
|
||||
|
|
|
@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
|
|||
*/
|
||||
void dumpString(std::string_view s, Sink & sink);
|
||||
|
||||
/**
|
||||
* If the NAR archive contains a single file at top-level, then save
|
||||
* the contents of the file to `s`. Otherwise barf.
|
||||
*/
|
||||
struct RetrieveRegularNARSink : ParseSink
|
||||
{
|
||||
bool regular = true;
|
||||
Sink & sink;
|
||||
|
||||
RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
|
||||
|
||||
void createDirectory(const Path & path) override
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
|
||||
void receiveContents(std::string_view data) override
|
||||
{
|
||||
sink(data);
|
||||
}
|
||||
|
||||
void createSymlink(const Path & path, const std::string & target) override
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
};
|
||||
|
||||
void parseDump(ParseSink & sink, Source & source);
|
||||
|
||||
void restorePath(const Path & path, Source & source);
|
||||
|
|
|
@ -5,6 +5,54 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
void copyRecursive(
|
||||
SourceAccessor & accessor, const CanonPath & from,
|
||||
ParseSink & sink, const Path & to)
|
||||
{
|
||||
auto stat = accessor.lstat(from);
|
||||
|
||||
switch (stat.type) {
|
||||
case SourceAccessor::tSymlink:
|
||||
{
|
||||
sink.createSymlink(to, accessor.readLink(from));
|
||||
}
|
||||
|
||||
case SourceAccessor::tRegular:
|
||||
{
|
||||
sink.createRegularFile(to);
|
||||
if (stat.isExecutable)
|
||||
sink.isExecutable();
|
||||
LambdaSink sink2 {
|
||||
[&](auto d) {
|
||||
sink.receiveContents(d);
|
||||
}
|
||||
};
|
||||
accessor.readFile(from, sink2, [&](uint64_t size) {
|
||||
sink.preallocateContents(size);
|
||||
});
|
||||
break;
|
||||
}
|
||||
|
||||
case SourceAccessor::tDirectory:
|
||||
{
|
||||
sink.createDirectory(to);
|
||||
for (auto & [name, _] : accessor.readDirectory(from)) {
|
||||
copyRecursive(
|
||||
accessor, from + name,
|
||||
sink, to + "/" + name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
case SourceAccessor::tMisc:
|
||||
throw Error("file '%1%' has an unsupported type", from);
|
||||
|
||||
default:
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
struct RestoreSinkSettings : Config
|
||||
{
|
||||
Setting<bool> preallocateContents{this, false, "preallocate-contents",
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
#include "types.hh"
|
||||
#include "serialise.hh"
|
||||
#include "source-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
|
@ -11,32 +12,93 @@ namespace nix {
|
|||
*/
|
||||
struct ParseSink
|
||||
{
|
||||
virtual void createDirectory(const Path & path) { };
|
||||
virtual void createDirectory(const Path & path) = 0;
|
||||
|
||||
virtual void createRegularFile(const Path & path) { };
|
||||
virtual void closeRegularFile() { };
|
||||
virtual void isExecutable() { };
|
||||
virtual void createRegularFile(const Path & path) = 0;
|
||||
virtual void receiveContents(std::string_view data) = 0;
|
||||
virtual void isExecutable() = 0;
|
||||
virtual void closeRegularFile() = 0;
|
||||
|
||||
virtual void createSymlink(const Path & path, const std::string & target) = 0;
|
||||
|
||||
/**
|
||||
* An optimization. By default, do nothing.
|
||||
*/
|
||||
virtual void preallocateContents(uint64_t size) { };
|
||||
virtual void receiveContents(std::string_view data) { };
|
||||
|
||||
virtual void createSymlink(const Path & path, const std::string & target) { };
|
||||
};
|
||||
|
||||
/**
|
||||
* Recusively copy file system objects from the source into the sink.
|
||||
*/
|
||||
void copyRecursive(
|
||||
SourceAccessor & accessor, const CanonPath & sourcePath,
|
||||
ParseSink & sink, const Path & destPath);
|
||||
|
||||
/**
|
||||
* Ignore everything and do nothing
|
||||
*/
|
||||
struct NullParseSink : ParseSink
|
||||
{
|
||||
void createDirectory(const Path & path) override { }
|
||||
void receiveContents(std::string_view data) override { }
|
||||
void createSymlink(const Path & path, const std::string & target) override { }
|
||||
void createRegularFile(const Path & path) override { }
|
||||
void closeRegularFile() override { }
|
||||
void isExecutable() override { }
|
||||
};
|
||||
|
||||
/**
|
||||
* Write files at the given path
|
||||
*/
|
||||
struct RestoreSink : ParseSink
|
||||
{
|
||||
Path dstPath;
|
||||
AutoCloseFD fd;
|
||||
|
||||
|
||||
void createDirectory(const Path & path) override;
|
||||
|
||||
void createRegularFile(const Path & path) override;
|
||||
void closeRegularFile() override;
|
||||
void isExecutable() override;
|
||||
void preallocateContents(uint64_t size) override;
|
||||
void receiveContents(std::string_view data) override;
|
||||
void isExecutable() override;
|
||||
void closeRegularFile() override;
|
||||
|
||||
void createSymlink(const Path & path, const std::string & target) override;
|
||||
|
||||
void preallocateContents(uint64_t size) override;
|
||||
|
||||
private:
|
||||
AutoCloseFD fd;
|
||||
};
|
||||
|
||||
/**
|
||||
* Restore a single file at the top level, passing along
|
||||
* `receiveContents` to the underlying `Sink`. For anything but a single
|
||||
* file, set `regular = true` so the caller can fail accordingly.
|
||||
*/
|
||||
struct RegularFileSink : ParseSink
|
||||
{
|
||||
bool regular = true;
|
||||
Sink & sink;
|
||||
|
||||
RegularFileSink(Sink & sink) : sink(sink) { }
|
||||
|
||||
void createDirectory(const Path & path) override
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
|
||||
void receiveContents(std::string_view data) override
|
||||
{
|
||||
sink(data);
|
||||
}
|
||||
|
||||
void createSymlink(const Path & path, const std::string & target) override
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
|
||||
void createRegularFile(const Path & path) override { }
|
||||
void closeRegularFile() override { }
|
||||
void isExecutable() override { }
|
||||
};
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue