mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-09-20 16:38:05 +03:00
SourceAccessor::readFile(): Support reading into a sink
This commit is contained in:
parent
7a086a32bc
commit
57db3be9e4
5 changed files with 94 additions and 13 deletions
|
@ -19,11 +19,14 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string readFile(const CanonPath & path) override
|
void readFile(
|
||||||
|
const CanonPath & path,
|
||||||
|
Sink & sink,
|
||||||
|
std::function<void(uint64_t)> sizeCallback) override
|
||||||
{
|
{
|
||||||
auto absPath = makeAbsPath(path);
|
auto absPath = makeAbsPath(path);
|
||||||
checkAllowed(absPath);
|
checkAllowed(absPath);
|
||||||
return PosixSourceAccessor::readFile(absPath);
|
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool pathExists(const CanonPath & path) override
|
bool pathExists(const CanonPath & path) override
|
||||||
|
|
|
@ -17,7 +17,7 @@ StorePath InputAccessor::fetchToStore(
|
||||||
if (method == FileIngestionMethod::Recursive)
|
if (method == FileIngestionMethod::Recursive)
|
||||||
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
|
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
|
||||||
else
|
else
|
||||||
sink(readFile(path)); // FIXME: stream
|
readFile(path, sink);
|
||||||
});
|
});
|
||||||
|
|
||||||
auto storePath =
|
auto storePath =
|
||||||
|
|
|
@ -44,12 +44,15 @@ void SourceAccessor::dumpPath(
|
||||||
{
|
{
|
||||||
auto dumpContents = [&](const CanonPath & path)
|
auto dumpContents = [&](const CanonPath & path)
|
||||||
{
|
{
|
||||||
/* It would be nice if this was streaming, but we need the
|
sink << "contents";
|
||||||
size before the contents. */
|
std::optional<uint64_t> size;
|
||||||
auto s = readFile(path);
|
readFile(path, sink, [&](uint64_t _size)
|
||||||
sink << "contents" << s.size();
|
{
|
||||||
sink(s);
|
size = _size;
|
||||||
writePadding(s.size(), sink);
|
sink << _size;
|
||||||
|
});
|
||||||
|
assert(size);
|
||||||
|
writePadding(*size, sink);
|
||||||
};
|
};
|
||||||
|
|
||||||
std::function<void(const CanonPath & path)> dump;
|
std::function<void(const CanonPath & path)> dump;
|
||||||
|
|
|
@ -10,6 +10,28 @@ SourceAccessor::SourceAccessor()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string SourceAccessor::readFile(const CanonPath & path)
|
||||||
|
{
|
||||||
|
StringSink sink;
|
||||||
|
std::optional<uint64_t> size;
|
||||||
|
readFile(path, sink, [&](uint64_t _size)
|
||||||
|
{
|
||||||
|
size = _size;
|
||||||
|
});
|
||||||
|
assert(size && *size == sink.s.size());
|
||||||
|
return std::move(sink.s);
|
||||||
|
}
|
||||||
|
|
||||||
|
void SourceAccessor::readFile(
|
||||||
|
const CanonPath & path,
|
||||||
|
Sink & sink,
|
||||||
|
std::function<void(uint64_t)> sizeCallback)
|
||||||
|
{
|
||||||
|
auto s = readFile(path);
|
||||||
|
sizeCallback(s.size());
|
||||||
|
sink(s);
|
||||||
|
}
|
||||||
|
|
||||||
Hash SourceAccessor::hashPath(
|
Hash SourceAccessor::hashPath(
|
||||||
const CanonPath & path,
|
const CanonPath & path,
|
||||||
PathFilter & filter,
|
PathFilter & filter,
|
||||||
|
@ -33,9 +55,41 @@ std::string SourceAccessor::showPath(const CanonPath & path)
|
||||||
return path.abs();
|
return path.abs();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string PosixSourceAccessor::readFile(const CanonPath & path)
|
void PosixSourceAccessor::readFile(
|
||||||
|
const CanonPath & path,
|
||||||
|
Sink & sink,
|
||||||
|
std::function<void(uint64_t)> sizeCallback)
|
||||||
{
|
{
|
||||||
return nix::readFile(path.abs());
|
// FIXME: add O_NOFOLLOW since symlinks should be resolved by the
|
||||||
|
// caller?
|
||||||
|
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
||||||
|
if (!fd)
|
||||||
|
throw SysError("opening file '%1%'", path);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
if (fstat(fd.get(), &st) == -1)
|
||||||
|
throw SysError("statting file");
|
||||||
|
|
||||||
|
sizeCallback(st.st_size);
|
||||||
|
|
||||||
|
off_t left = st.st_size;
|
||||||
|
|
||||||
|
std::vector<unsigned char> buf(64 * 1024);
|
||||||
|
while (left) {
|
||||||
|
checkInterrupt();
|
||||||
|
ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size()));
|
||||||
|
if (rd == -1) {
|
||||||
|
if (errno != EINTR)
|
||||||
|
throw SysError("reading from file '%s'", showPath(path));
|
||||||
|
}
|
||||||
|
else if (rd == 0)
|
||||||
|
throw SysError("unexpected end-of-file reading '%s'", showPath(path));
|
||||||
|
else {
|
||||||
|
assert(rd <= left);
|
||||||
|
sink({(char *) buf.data(), (size_t) rd});
|
||||||
|
left -= rd;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool PosixSourceAccessor::pathExists(const CanonPath & path)
|
bool PosixSourceAccessor::pathExists(const CanonPath & path)
|
||||||
|
|
|
@ -5,6 +5,8 @@
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
struct Sink;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A read-only filesystem abstraction. This is used by the Nix
|
* A read-only filesystem abstraction. This is used by the Nix
|
||||||
* evaluator and elsewhere for accessing sources in various
|
* evaluator and elsewhere for accessing sources in various
|
||||||
|
@ -20,7 +22,23 @@ struct SourceAccessor
|
||||||
virtual ~SourceAccessor()
|
virtual ~SourceAccessor()
|
||||||
{ }
|
{ }
|
||||||
|
|
||||||
virtual std::string readFile(const CanonPath & path) = 0;
|
/**
|
||||||
|
* Return the contents of a file as a string.
|
||||||
|
*/
|
||||||
|
virtual std::string readFile(const CanonPath & path);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the contents of a file as a sink. `sizeCallback` must be
|
||||||
|
* called with the size of the file before any data is written to
|
||||||
|
* the sink.
|
||||||
|
*
|
||||||
|
* Note: subclasses of `SourceAccessor` need to implement at least
|
||||||
|
* one of the `readFile()` variants.
|
||||||
|
*/
|
||||||
|
virtual void readFile(
|
||||||
|
const CanonPath & path,
|
||||||
|
Sink & sink,
|
||||||
|
std::function<void(uint64_t)> sizeCallback = [](uint64_t size){});
|
||||||
|
|
||||||
virtual bool pathExists(const CanonPath & path) = 0;
|
virtual bool pathExists(const CanonPath & path) = 0;
|
||||||
|
|
||||||
|
@ -97,7 +115,10 @@ struct PosixSourceAccessor : SourceAccessor
|
||||||
*/
|
*/
|
||||||
time_t mtime = 0;
|
time_t mtime = 0;
|
||||||
|
|
||||||
std::string readFile(const CanonPath & path) override;
|
void readFile(
|
||||||
|
const CanonPath & path,
|
||||||
|
Sink & sink,
|
||||||
|
std::function<void(uint64_t)> sizeCallback) override;
|
||||||
|
|
||||||
bool pathExists(const CanonPath & path) override;
|
bool pathExists(const CanonPath & path) override;
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue