Merge pull request #11171 from DeterminateSystems/speed-up-tarball-downloads

Increase download buffer size and improve tarball import logging

src/libfetchers/github.cc

@@ -254,10 +254,15 @@ struct GitArchiveInputScheme : InputScheme
             getFileTransfer()->download(std::move(req), sink);
         });
 
+        auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
+            fmt("unpacking '%s' into the Git cache", input.to_string()));
+
         TarArchive archive { *source };
         auto parseSink = getTarballCache()->getFileSystemObjectSink();
         auto lastModified = unpackTarfileToSink(archive, *parseSink);
 
+        act.reset();
+
         TarballInfo tarballInfo {
             .treeHash = parseSink->sync(),
             .lastModified = lastModified

src/libfetchers/tarball.cc

@@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball(
 
     // TODO: fall back to cached value if download fails.
 
+    auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
+        fmt("unpacking '%s' into the Git cache", url));
+
     AutoDelete cleanupTemp;
 
     /* Note: if the download is cached, `importTarball()` will receive
@@ -167,6 +170,8 @@ DownloadTarballResult downloadTarball(
     auto parseSink = getTarballCache()->getFileSystemObjectSink();
     auto lastModified = unpackTarfileToSink(archive, *parseSink);
 
+    act.reset();
+
     auto res(_res->lock());
 
     Attrs infoAttrs;

src/libstore/filetransfer.cc

@@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer
 
         curl_off_t writtenToSink = 0;
 
+        std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
+
         inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
+
         /* Get the HTTP status code, or 0 for other protocols. */
         long getHTTPStatus()
         {
@@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer
 
         void finish(CURLcode code)
         {
+            auto finishTime = std::chrono::steady_clock::now();
+
             auto httpStatus = getHTTPStatus();
 
-            debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
+            debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
-                request.verb(), request.uri, code, httpStatus, result.bodySize);
+                request.verb(), request.uri, code, httpStatus, result.bodySize,
+                std::chrono::duration_cast<std::chrono::milliseconds>(finishTime - startTime).count() / 1000.0f
+                );
 
             appendCurrentUrl();
 
@@ -851,8 +858,10 @@ void FileTransfer::download(
            buffer). We don't wait forever to prevent stalling the
            download thread. (Hopefully sleeping will throttle the
            sender.) */
-        if (state->data.size() > 1024 * 1024) {
+        if (state->data.size() > fileTransferSettings.downloadBufferSize) {
             debug("download buffer is full; going to sleep");
+            static bool haveWarned = false;
+            warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
             state.wait_for(state->request, std::chrono::seconds(10));
         }
 

src/libstore/filetransfer.hh

@@ -47,6 +47,12 @@ struct FileTransferSettings : Config
 
     Setting<unsigned int> tries{this, 5, "download-attempts",
         "How often Nix will attempt to download a file before giving up."};
+
+    Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
+        R"(
+          The size of Nix's internal download buffer during `curl` transfers. If data is
+          not processed quickly enough to exceed the size of this buffer, downloads may stall.
+        )"};
 };
 
 extern FileTransferSettings fileTransferSettings;