Merge remote-tracking branch 'origin/master' into flakes

.version

@@ -1 +1 @@
-2.3
+2.4

doc/manual/expressions/builtins.xml

@@ -448,7 +448,7 @@ stdenv.mkDerivation { … }
       <example>
         <title>Fetching an arbitrary ref</title>
         <programlisting>builtins.fetchGit {
-  url = "https://gitub.com/NixOS/nix.git";
+  url = "https://github.com/NixOS/nix.git";
   ref = "refs/heads/0.5-release";
 }</programlisting>
       </example>
@@ -499,11 +499,8 @@ stdenv.mkDerivation { … }
         <title>Fetching a tag</title>
         <programlisting>builtins.fetchGit {
   url = "https://github.com/nixos/nix.git";
-  ref = "tags/1.9";
+  ref = "refs/tags/1.9";
 }</programlisting>
-        <note><para>Due to a bug (<link
-        xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>),
-        only non-annotated tags can be fetched.</para></note>
       </example>
 
       <example>

doc/manual/release-notes/rl-2.3.xml

@@ -13,9 +13,8 @@ incompatible changes:</para>
 
   <listitem>
     <para>Nix now uses BSD file locks instead of POSIX file
-    locks. Since previous releases used POSIX file locks, you should
+    locks. Because of this, you should not use Nix 2.3 and previous
-    not use Nix 2.2 and previous releases at the same time on a Nix
+    releases at the same time on a Nix store.</para>
-    store.</para>
   </listitem>
 
 </itemizedlist>
@@ -47,9 +46,9 @@ incompatible changes:</para>
   </listitem>
 
   <listitem>
-    <para><command>nix</command>: Add
+    <para>The <command>nix</command> command has a new
     <option>--print-build-logs</option> (<option>-L</option>) flag to
-    print build log output to stderr rather than showing the last log
+    print build log output to stderr, rather than showing the last log
     line in the progress bar. To distinguish between concurrent
     builds, log lines are prefixed by the name of the package.
     </para>
@@ -57,7 +56,7 @@ incompatible changes:</para>
 
   <listitem>
     <para>Builds are now executed in a pseudo-terminal, and the
-    <envar>TERM</envar> evnironment variable is set to
+    <envar>TERM</envar> environment variable is set to
     <literal>xterm-256color</literal>. This allows many programs
     (e.g. <command>gcc</command>, <command>clang</command>,
     <command>cmake</command>) to print colorized log output.</para>

maintainers/upload-release.pl

@@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n";
 
 File::Path::make_path($releasesDir);
 if (system("mountpoint -q $releasesDir") != 0) {
-    system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die;
+    system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die;
 }
 
 my $releaseDir = "$releasesDir/nix/$releaseName";

src/libstore/binary-cache-store.cc

@@ -10,10 +10,13 @@
 #include "nar-info-disk-cache.hh"
 #include "nar-accessor.hh"
 #include "json.hh"
+#include "thread-pool.hh"
 
 #include <chrono>
-
 #include <future>
+#include <regex>
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
 
     auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
 
+    auto narAccessor = makeNarAccessor(nar);
+
+    if (accessor_)
+        accessor_->addToCache(info.path, *nar, narAccessor);
+
     /* Optionally write a JSON file containing a listing of the
        contents of the NAR. */
     if (writeNARListing) {
@@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
             JSONObject jsonRoot(jsonOut);
             jsonRoot.attr("version", 1);
 
-            auto narAccessor = makeNarAccessor(nar);
-
-            if (accessor_)
-                accessor_->addToCache(info.path, *nar, narAccessor);
-
             {
                 auto res = jsonRoot.placeholder("root");
                 listNar(res, narAccessor, "", true);
@@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
         upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json");
     }
 
-    else {
-        if (accessor_)
-            accessor_->addToCache(info.path, *nar, makeNarAccessor(nar));
-    }
-
     /* Compress the NAR. */
     narInfo->compression = compression;
     auto now1 = std::chrono::steady_clock::now();
@@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
         % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
         % duration);
 
-    /* Atomically write the NAR file. */
     narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
         + (compression == "xz" ? ".xz" :
            compression == "bzip2" ? ".bz2" :
            compression == "br" ? ".br" :
            "");
+
+    /* Optionally maintain an index of DWARF debug info files
+       consisting of JSON files named 'debuginfo/<build-id>' that
+       specify the NAR file and member containing the debug info. */
+    if (writeDebugInfo) {
+
+        std::string buildIdDir = "/lib/debug/.build-id";
+
+        if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) {
+
+            ThreadPool threadPool(25);
+
+            auto doFile = [&](std::string member, std::string key, std::string target) {
+                checkInterrupt();
+
+                nlohmann::json json;
+                json["archive"] = target;
+                json["member"] = member;
+
+                // FIXME: or should we overwrite? The previous link may point
+                // to a GC'ed file, so overwriting might be useful...
+                if (fileExists(key)) return;
+
+                printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target);
+
+                upsertFile(key, json.dump(), "application/json");
+            };
+
+            std::regex regex1("^[0-9a-f]{2}$");
+            std::regex regex2("^[0-9a-f]{38}\\.debug$");
+
+            for (auto & s1 : narAccessor->readDirectory(buildIdDir)) {
+                auto dir = buildIdDir + "/" + s1;
+
+                if (narAccessor->stat(dir).type != FSAccessor::tDirectory
+                    || !std::regex_match(s1, regex1))
+                    continue;
+
+                for (auto & s2 : narAccessor->readDirectory(dir)) {
+                    auto debugPath = dir + "/" + s2;
+
+                    if (narAccessor->stat(debugPath).type != FSAccessor::tRegular
+                        || !std::regex_match(s2, regex2))
+                        continue;
+
+                    auto buildId = s1 + s2;
+
+                    std::string key = "debuginfo/" + buildId;
+                    std::string target = "../" + narInfo->url;
+
+                    threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target));
+                }
+            }
+
+            threadPool.process();
+        }
+    }
+
+    /* Atomically write the NAR file. */
     if (repair || !fileExists(narInfo->url)) {
         stats.narWrite++;
         upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");

src/libstore/binary-cache-store.hh

@@ -17,6 +17,7 @@ public:
 
     const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
     const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
+    const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"};
     const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
     const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
     const Setting<bool> parallelCompression{this, false, "parallel-compression",

src/libstore/local-binary-cache-store.cc

@@ -63,6 +63,8 @@ protected:
 void LocalBinaryCacheStore::init()
 {
     createDirs(binaryCacheDir + "/nar");
+    if (writeDebugInfo)
+        createDirs(binaryCacheDir + "/debuginfo");
     BinaryCacheStore::init();
 }