#include "serialise.hh" #include "util.hh" #include #include #include #include namespace nix { void BufferedSink::operator () (const unsigned char * data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); while (len) { /* Optimisation: bypass the buffer if the data exceeds the buffer size. */ if (bufPos + len >= bufSize) { flush(); write(data, len); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer when it's full. */ size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; memcpy(buffer.get() + bufPos, data, n); data += n; bufPos += n; len -= n; if (bufPos == bufSize) flush(); } } void BufferedSink::flush() { if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() write(buffer.get(), n); } FdSink::~FdSink() { try { flush(); } catch (...) { ignoreException(); } } size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { logWarning({ .name = "Large path", .description = "dumping very large path (> 256 MiB); this may run out of memory" }); } void FdSink::write(const unsigned char * data, size_t len) { written += len; static bool warned = false; if (warn && !warned) { if (written > threshold) { warnLargeDump(); warned = true; } } try { writeFull(fd, data, len); } catch (SysError & e) { _good = false; throw; } } bool FdSink::good() { return _good; } void Source::operator () (unsigned char * data, size_t len) { while (len) { size_t n = read(data, len); data += n; len -= n; } } std::string Source::drain() { std::string s; std::vector buf(8192); while (true) { size_t n; try { n = read(buf.data(), buf.size()); s.append((char *) buf.data(), n); } catch (EndOfFile &) { break; } } return s; } size_t BufferedSource::read(unsigned char * data, size_t len) { if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); /* Copy out the data in the buffer. */ size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; memcpy(data, buffer.get() + bufPosOut, n); bufPosOut += n; if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; return n; } bool BufferedSource::hasData() { return bufPosOut < bufPosIn; } size_t FdSource::readUnbuffered(unsigned char * data, size_t len) { ssize_t n; do { checkInterrupt(); n = ::read(fd, (char *) data, len); } while (n == -1 && errno == EINTR); if (n == -1) { _good = false; throw SysError("reading from file"); } if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } read += n; return n; } bool FdSource::good() { return _good; } size_t StringSource::read(unsigned char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); size_t n = s.copy((char *) data, len, pos); pos += n; return n; } #if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 #error Coroutines are broken in this version of Boost! #endif std::unique_ptr sinkToSource( std::function fun, std::function eof) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine> coro_t; std::function fun; std::function eof; std::optional coro; bool started = false; /* It would be nicer to have the co-routines have both args and a return value, but unfortunately that was removed from Boost's implementation for some reason, so we use some extra state instead. */ size_t wanted = 0; SinkToSource(std::function fun, std::function eof) : fun(fun), eof(eof) { } std::basic_string cur; size_t pos = 0; size_t read(unsigned char * data, size_t len) override { wanted = len < cur.size() ? 0 : len - cur.size(); if (!coro) { coro = coro_t::pull_type([&](coro_t::push_type & yield) { LambdaSink sink([&](const uint8_t * data, size_t len) { if (len) yield(std::basic_string { data, len }); }); fun(sink, wanted); }); } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { if (!cur.empty()) (*coro)(); cur = coro->get(); pos = 0; } auto numCopied = cur.copy(data, len, pos); pos += numCopied; return numCopied; } }; return std::make_unique(fun, eof); } void writePadding(size_t len, Sink & sink) { if (len % 8) { unsigned char zero[8]; memset(zero, 0, sizeof(zero)); sink(zero, 8 - (len % 8)); } } void writeString(const unsigned char * buf, size_t len, Sink & sink) { sink << len; sink(buf, len); writePadding(len, sink); } Sink & operator << (Sink & sink, const string & s) { writeString((const unsigned char *) s.data(), s.size(), sink); return sink; } template void writeStrings(const T & ss, Sink & sink) { sink << ss.size(); for (auto & i : ss) sink << i; } Sink & operator << (Sink & sink, const Strings & s) { writeStrings(s, sink); return sink; } Sink & operator << (Sink & sink, const StringSet & s) { writeStrings(s, sink); return sink; } void readPadding(size_t len, Source & source) { if (len % 8) { unsigned char zero[8]; size_t n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) if (zero[i]) throw SerialisationError("non-zero padding"); } } size_t readString(unsigned char * buf, size_t max, Source & source) { auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } string readString(Source & source, size_t max) { auto len = readNum(source); if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); source((unsigned char*) res.data(), len); readPadding(len, source); return res; } Source & operator >> (Source & in, string & s) { s = readString(in); return in; } template T readStrings(Source & source) { auto count = readNum(source); T ss; while (count--) ss.insert(ss.end(), readString(source)); return ss; } template Paths readStrings(Source & source); template PathSet readStrings(Source & source); void StringSink::operator () (const unsigned char * data, size_t len) { static bool warned = false; if (!warned && s->size() > threshold) { warnLargeDump(); warned = true; } s->append((const char *) data, len); } size_t ChainSource::read(unsigned char * data, size_t len) { if (useSecond) { return source2.read(data, len); } else { try { return source1.read(data, len); } catch (EndOfFile &) { useSecond = true; return this->read(data, len); } } } }