Remove the "locked" flag from the fetcher cache

This also reworks the Mercurial fetcher (which was still using the
old cache interface) to have two distinct cache mappings:

* A ref-to-rev mapping, which is store-independent.
* A rev-to-store-path mapping.

src/libfetchers/cache.cc

@@ -14,7 +14,7 @@ create table if not exists Cache (
     input     text not null,
     info      text not null,
     path      text not null,
-    immutable integer not null,
+    immutable integer not null, /* obsolete */
     timestamp integer not null,
     primary key (input)
 );
@@ -45,7 +45,7 @@ struct CacheImpl : Cache
         state->db.exec(schema);
 
         state->add.create(state->db,
-            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)");
+            "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, false, ?)");
 
         state->lookup.create(state->db,
             "select info, path, immutable, timestamp from Cache where input = ?");
@@ -59,7 +59,6 @@ struct CacheImpl : Cache
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
             ("") // no path
-            (false)
             (time(0)).exec();
     }
 
@@ -109,14 +108,12 @@ struct CacheImpl : Cache
         Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
-        const StorePath & storePath,
+        const StorePath & storePath) override
-        bool locked) override
     {
         _state.lock()->add.use()
             (attrsToJSON(inAttrs).dump())
             (attrsToJSON(infoAttrs).dump())
             (store.printStorePath(storePath))
-            (locked)
             (time(0)).exec();
     }
 

src/libfetchers/cache.hh

@@ -53,8 +53,7 @@ struct Cache
         Store & store,
         const Attrs & inAttrs,
         const Attrs & infoAttrs,
-        const StorePath & storePath,
+        const StorePath & storePath) = 0;
-        bool locked) = 0;
 
     virtual std::optional<std::pair<Attrs, StorePath>> lookup(
         Store & store,

src/libfetchers/fetch-to-store.cc

@@ -47,10 +47,9 @@ StorePath fetchToStore(
             name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
 
     if (cacheKey && mode == FetchMode::Copy)
-        fetchers::getCache()->add(store, *cacheKey, {}, storePath, true);
+        fetchers::getCache()->add(store, *cacheKey, {}, storePath);
 
     return storePath;
 }
 
-
 }

src/libfetchers/tarball.cc

@@ -99,8 +99,7 @@ DownloadFileResult downloadFile(
             *store,
             inAttrs,
             infoAttrs,
-            *storePath,
+            *storePath);
-            false);
     }
 
     return {

src/libfetchers/unix/mercurial.cc

@@ -224,22 +224,17 @@ struct MercurialInputScheme : InputScheme
 
         if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
 
-        auto checkHashAlgorithm = [&](const std::optional<Hash> & hash)
+        auto revInfoCacheKey = [&](const Hash & rev)
         {
-            if (hash.has_value() && hash->algo != HashAlgorithm::SHA1)
+            if (rev.algo != HashAlgorithm::SHA1)
-                throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
+                throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.to_string(HashFormat::Base16, true));
-        };
 
-
+            return Attrs{
-        auto getLockedAttrs = [&]()
+                {"_what", "hgRev"},
-        {
+                {"store", store->storeDir},
-            checkHashAlgorithm(input.getRev());
-
-            return Attrs({
-                {"type", "hg"},
                 {"name", name},
-                {"rev", input.getRev()->gitRev()},
+                {"rev", input.getRev()->gitRev()}
-            });
+            };
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath
@@ -250,26 +245,22 @@ struct MercurialInputScheme : InputScheme
             return storePath;
         };
 
-        if (input.getRev()) {
+        /* Check the cache for the most recent rev for this URL/ref. */
-            if (auto res = getCache()->lookup(*store, getLockedAttrs()))
+        Attrs refToRevCacheKey{
-                return makeResult(res->first, std::move(res->second));
+            {"_what", "hgRefToRev"},
+            {"url", actualUrl},
+            {"ref", *input.getRef()}
+        };
+
+        if (!input.getRev()) {
+            if (auto res = getCache()->lookupWithTTL(refToRevCacheKey))
+                input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
         }
 
-        auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
+        /* If we have a rev, check if we have a cached store path. */
-
+        if (auto rev = input.getRev()) {
-        Attrs unlockedAttrs({
+            if (auto res = getCache()->lookupExpired(*store, revInfoCacheKey(*rev)))
-            {"type", "hg"},
+                return makeResult(res->infoAttrs, res->storePath);
-            {"name", name},
-            {"url", actualUrl},
-            {"ref", *input.getRef()},
-        });
-
-        if (auto res = getCache()->lookup(*store, unlockedAttrs)) {
-            auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1);
-            if (!input.getRev() || input.getRev() == rev2) {
-                input.attrs.insert_or_assign("rev", rev2.gitRev());
-                return makeResult(res->first, std::move(res->second));
-            }
         }
 
         Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false));
@@ -302,21 +293,29 @@ struct MercurialInputScheme : InputScheme
             }
         }
 
+        /* Fetch the remote rev or ref. */
         auto tokens = tokenizeString<std::vector<std::string>>(
-            runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+            runHg({
+                "log", "-R", cacheDir,
+                "-r", input.getRev() ? input.getRev()->gitRev() : *input.getRef(),
+                "--template", "{node} {rev} {branch}"
+            }));
         assert(tokens.size() == 3);
 
-        input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev());
+        auto rev = Hash::parseAny(tokens[0], HashAlgorithm::SHA1);
+        input.attrs.insert_or_assign("rev", rev.gitRev());
         auto revCount = std::stoull(tokens[1]);
         input.attrs.insert_or_assign("ref", tokens[2]);
 
-        if (auto res = getCache()->lookup(*store, getLockedAttrs()))
+        /* Now that we have the rev, check the cache again for a
-            return makeResult(res->first, std::move(res->second));
+           cached store path. */
+        if (auto res = getCache()->lookupExpired(*store, revInfoCacheKey(rev)))
+            return makeResult(res->infoAttrs, res->storePath);
 
         Path tmpDir = createTempDir();
         AutoDelete delTmpDir(tmpDir, true);
 
-        runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
+        runHg({ "archive", "-R", cacheDir, "-r", rev.gitRev(), tmpDir });
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
@@ -324,24 +323,17 @@ struct MercurialInputScheme : InputScheme
         auto storePath = store->addToStore(name, accessor, CanonPath { tmpDir });
 
         Attrs infoAttrs({
-            {"rev", input.getRev()->gitRev()},
             {"revCount", (uint64_t) revCount},
         });
 
         if (!origRev)
-            getCache()->add(
+            getCache()->upsert(refToRevCacheKey, {{"rev", rev.gitRev()}});
-                *store,
-                unlockedAttrs,
-                infoAttrs,
-                storePath,
-                false);
 
         getCache()->add(
             *store,
-            getLockedAttrs(),
+            revInfoCacheKey(rev),
             infoAttrs,
-            storePath,
+            storePath);
-            true);
 
         return makeResult(infoAttrs, std::move(storePath));
     }

tests/functional/fetchMercurial.sh

@@ -101,6 +101,7 @@ path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file:
 [[ $path2 = $path4 ]]
 
 echo paris > $repo/hello
+
 # Passing a `name` argument should be reflected in the output path
 path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath")
 [[ $path5 =~ -foo$ ]]