max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-22 05:56:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11171 from DeterminateSystems/speed-up-tarball-downloads

Browse source 
Increase download buffer size and improve tarball import logging
This commit is contained in:
Robert Hensing 2024-07-29 15:02:35 +02:00 committed by GitHub
commit 6e3bba5e26
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 28 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/libfetchers/github.cc
View file

@ -254,10 +254,15 @@ struct GitArchiveInputScheme : InputScheme
getFileTransfer()->download(std::move(req), sink);
});
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
TarballInfo tarballInfo {
.treeHash = parseSink->sync(),
.lastModified = lastModified

5
src/libfetchers/tarball.cc
View file

@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball(
// TODO: fall back to cached value if download fails.
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", url));
AutoDelete cleanupTemp;
/* Note: if the download is cached, `importTarball()` will receive
@ -167,6 +170,8 @@ DownloadTarballResult downloadTarball(
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
auto res(_res->lock());
Attrs infoAttrs;

15
src/libstore/filetransfer.cc
View file

@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer
void finish(CURLcode code)
{
auto finishTime = std::chrono::steady_clock::now();
auto httpStatus = getHTTPStatus();
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
request.verb(), request.uri, code, httpStatus, result.bodySize);
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
request.verb(), request.uri, code, httpStatus, result.bodySize,
std::chrono::duration_cast<std::chrono::milliseconds>(finishTime - startTime).count() / 1000.0f
);
appendCurrentUrl();
@ -851,8 +858,10 @@ void FileTransfer::download(
buffer). We don't wait forever to prevent stalling the
download thread. (Hopefully sleeping will throttle the
sender.) */
if (state->data.size() > 1024 * 1024) {
if (state->data.size() > fileTransferSettings.downloadBufferSize) {
debug("download buffer is full; going to sleep");
static bool haveWarned = false;
warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
state.wait_for(state->request, std::chrono::seconds(10));
}

6
src/libstore/filetransfer.hh
View file

@ -47,6 +47,12 @@ struct FileTransferSettings : Config
Setting<unsigned int> tries{this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."};
Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
R"(
The size of Nix's internal download buffer during `curl` transfers. If data is
not processed quickly enough to exceed the size of this buffer, downloads may stall.
)"};
};
extern FileTransferSettings fileTransferSettings;