mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-22 22:16:16 +02:00
Merge pull request #11171 from DeterminateSystems/speed-up-tarball-downloads
Increase download buffer size and improve tarball import logging
This commit is contained in:
commit
6e3bba5e26
4 changed files with 28 additions and 3 deletions
|
@ -254,10 +254,15 @@ struct GitArchiveInputScheme : InputScheme
|
||||||
getFileTransfer()->download(std::move(req), sink);
|
getFileTransfer()->download(std::move(req), sink);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
|
||||||
|
fmt("unpacking '%s' into the Git cache", input.to_string()));
|
||||||
|
|
||||||
TarArchive archive { *source };
|
TarArchive archive { *source };
|
||||||
auto parseSink = getTarballCache()->getFileSystemObjectSink();
|
auto parseSink = getTarballCache()->getFileSystemObjectSink();
|
||||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||||
|
|
||||||
|
act.reset();
|
||||||
|
|
||||||
TarballInfo tarballInfo {
|
TarballInfo tarballInfo {
|
||||||
.treeHash = parseSink->sync(),
|
.treeHash = parseSink->sync(),
|
||||||
.lastModified = lastModified
|
.lastModified = lastModified
|
||||||
|
|
|
@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball(
|
||||||
|
|
||||||
// TODO: fall back to cached value if download fails.
|
// TODO: fall back to cached value if download fails.
|
||||||
|
|
||||||
|
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
|
||||||
|
fmt("unpacking '%s' into the Git cache", url));
|
||||||
|
|
||||||
AutoDelete cleanupTemp;
|
AutoDelete cleanupTemp;
|
||||||
|
|
||||||
/* Note: if the download is cached, `importTarball()` will receive
|
/* Note: if the download is cached, `importTarball()` will receive
|
||||||
|
@ -167,6 +170,8 @@ DownloadTarballResult downloadTarball(
|
||||||
auto parseSink = getTarballCache()->getFileSystemObjectSink();
|
auto parseSink = getTarballCache()->getFileSystemObjectSink();
|
||||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||||
|
|
||||||
|
act.reset();
|
||||||
|
|
||||||
auto res(_res->lock());
|
auto res(_res->lock());
|
||||||
|
|
||||||
Attrs infoAttrs;
|
Attrs infoAttrs;
|
||||||
|
|
|
@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
curl_off_t writtenToSink = 0;
|
curl_off_t writtenToSink = 0;
|
||||||
|
|
||||||
|
std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
|
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
|
||||||
|
|
||||||
/* Get the HTTP status code, or 0 for other protocols. */
|
/* Get the HTTP status code, or 0 for other protocols. */
|
||||||
long getHTTPStatus()
|
long getHTTPStatus()
|
||||||
{
|
{
|
||||||
|
@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer
|
||||||
|
|
||||||
void finish(CURLcode code)
|
void finish(CURLcode code)
|
||||||
{
|
{
|
||||||
|
auto finishTime = std::chrono::steady_clock::now();
|
||||||
|
|
||||||
auto httpStatus = getHTTPStatus();
|
auto httpStatus = getHTTPStatus();
|
||||||
|
|
||||||
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
|
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
|
||||||
request.verb(), request.uri, code, httpStatus, result.bodySize);
|
request.verb(), request.uri, code, httpStatus, result.bodySize,
|
||||||
|
std::chrono::duration_cast<std::chrono::milliseconds>(finishTime - startTime).count() / 1000.0f
|
||||||
|
);
|
||||||
|
|
||||||
appendCurrentUrl();
|
appendCurrentUrl();
|
||||||
|
|
||||||
|
@ -851,8 +858,10 @@ void FileTransfer::download(
|
||||||
buffer). We don't wait forever to prevent stalling the
|
buffer). We don't wait forever to prevent stalling the
|
||||||
download thread. (Hopefully sleeping will throttle the
|
download thread. (Hopefully sleeping will throttle the
|
||||||
sender.) */
|
sender.) */
|
||||||
if (state->data.size() > 1024 * 1024) {
|
if (state->data.size() > fileTransferSettings.downloadBufferSize) {
|
||||||
debug("download buffer is full; going to sleep");
|
debug("download buffer is full; going to sleep");
|
||||||
|
static bool haveWarned = false;
|
||||||
|
warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
|
||||||
state.wait_for(state->request, std::chrono::seconds(10));
|
state.wait_for(state->request, std::chrono::seconds(10));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -47,6 +47,12 @@ struct FileTransferSettings : Config
|
||||||
|
|
||||||
Setting<unsigned int> tries{this, 5, "download-attempts",
|
Setting<unsigned int> tries{this, 5, "download-attempts",
|
||||||
"How often Nix will attempt to download a file before giving up."};
|
"How often Nix will attempt to download a file before giving up."};
|
||||||
|
|
||||||
|
Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
|
||||||
|
R"(
|
||||||
|
The size of Nix's internal download buffer during `curl` transfers. If data is
|
||||||
|
not processed quickly enough to exceed the size of this buffer, downloads may stall.
|
||||||
|
)"};
|
||||||
};
|
};
|
||||||
|
|
||||||
extern FileTransferSettings fileTransferSettings;
|
extern FileTransferSettings fileTransferSettings;
|
||||||
|
|
Loading…
Reference in a new issue