diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 34eb6f32c..87a8e4702 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -51,57 +51,53 @@ struct CacheImpl : Cache } void upsert( - std::string_view domain, - const Attrs & key, + const Key & key, const Attrs & value) override { _state.lock()->upsert.use() - (domain) - (attrsToJSON(key).dump()) + (key.first) + (attrsToJSON(key.second).dump()) (attrsToJSON(value).dump()) (time(0)).exec(); } std::optional lookup( - std::string_view domain, - const Attrs & key) override + const Key & key) override { - if (auto res = lookupExpired(domain, key)) + if (auto res = lookupExpired(key)) return std::move(res->value); return {}; } std::optional lookupWithTTL( - std::string_view domain, - const Attrs & key) override + const Key & key) override { - if (auto res = lookupExpired(domain, key)) { + if (auto res = lookupExpired(key)) { if (!res->expired) return std::move(res->value); debug("ignoring expired cache entry '%s:%s'", - domain, attrsToJSON(key).dump()); + key.first, attrsToJSON(key.second).dump()); } return {}; } std::optional lookupExpired( - std::string_view domain, - const Attrs & key) override + const Key & key) override { auto state(_state.lock()); - auto keyJSON = attrsToJSON(key).dump(); + auto keyJSON = attrsToJSON(key.second).dump(); - auto stmt(state->lookup.use()(domain)(keyJSON)); + auto stmt(state->lookup.use()(key.first)(keyJSON)); if (!stmt.next()) { - debug("did not find cache entry for '%s:%s'", domain, keyJSON); + debug("did not find cache entry for '%s:%s'", key.first, keyJSON); return {}; } auto valueJSON = stmt.getStr(0); auto timestamp = stmt.getInt(1); - debug("using cache entry '%s:%s' -> '%s'", domain, keyJSON, valueJSON); + debug("using cache entry '%s:%s' -> '%s'", key.first, keyJSON, valueJSON); return Result { .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0), @@ -110,29 +106,27 @@ struct CacheImpl : Cache } void upsert( - std::string_view domain, - Attrs key, + Key key, Store & store, Attrs value, const StorePath & storePath) { /* Add the store prefix to the cache key to handle multiple store prefixes. */ - key.insert_or_assign("store", store.storeDir); + key.second.insert_or_assign("store", store.storeDir); value.insert_or_assign("storePath", (std::string) storePath.to_string()); - upsert(domain, key, value); + upsert(key, value); } std::optional lookupStorePath( - std::string_view domain, - Attrs key, + Key key, Store & store) override { - key.insert_or_assign("store", store.storeDir); + key.second.insert_or_assign("store", store.storeDir); - auto res = lookupExpired(domain, key); + auto res = lookupExpired(key); if (!res) return std::nullopt; auto storePathS = getStrAttr(res->value, "storePath"); @@ -143,14 +137,16 @@ struct CacheImpl : Cache store.addTempRoot(res2.storePath); if (!store.isValidPath(res2.storePath)) { // FIXME: we could try to substitute 'storePath'. - debug("ignoring disappeared cache entry '%s' -> '%s'", - attrsToJSON(key).dump(), + debug("ignoring disappeared cache entry '%s:%s' -> '%s'", + key.first, + attrsToJSON(key.second).dump(), store.printStorePath(res2.storePath)); return std::nullopt; } - debug("using cache entry '%s' -> '%s', '%s'", - attrsToJSON(key).dump(), + debug("using cache entry '%s:%s' -> '%s', '%s'", + key.first, + attrsToJSON(key.second).dump(), attrsToJSON(res2.value).dump(), store.printStorePath(res2.storePath)); @@ -158,11 +154,10 @@ struct CacheImpl : Cache } std::optional lookupStorePathWithTTL( - std::string_view domain, - Attrs key, + Key key, Store & store) override { - auto res = lookupStorePath(domain, std::move(key), store); + auto res = lookupStorePath(std::move(key), store); return res && !res->expired ? res : std::nullopt; } }; diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index 3295b56bc..1a72162d7 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -15,28 +15,35 @@ struct Cache virtual ~Cache() { } /** - * Add a value to the cache. The cache is an arbitrary mapping of - * Attrs to Attrs. + * A domain is a partition of the key/value cache for a particular + * purpose, e.g. "Git revision to revcount". + */ + using Domain = std::string_view; + + /** + * A cache key is a domain and an arbitrary set of attributes. + */ + using Key = std::pair; + + /** + * Add a key/value pair to the cache. */ virtual void upsert( - std::string_view domain, - const Attrs & key, + const Key & key, const Attrs & value) = 0; /** * Look up a key with infinite TTL. */ virtual std::optional lookup( - std::string_view domain, - const Attrs & key) = 0; + const Key & key) = 0; /** * Look up a key. Return nothing if its TTL has exceeded * `settings.tarballTTL`. */ virtual std::optional lookupWithTTL( - std::string_view domain, - const Attrs & key) = 0; + const Key & key) = 0; struct Result { @@ -49,8 +56,7 @@ struct Cache * exceeded `settings.tarballTTL`. */ virtual std::optional lookupExpired( - std::string_view domain, - const Attrs & key) = 0; + const Key & key) = 0; /** * Insert a cache entry that has a store path associated with @@ -58,8 +64,7 @@ struct Cache * associated store path is invalid. */ virtual void upsert( - std::string_view domain, - Attrs key, + Key key, Store & store, Attrs value, const StorePath & storePath) = 0; @@ -74,8 +79,7 @@ struct Cache * be valid, but it may be expired. */ virtual std::optional lookupStorePath( - std::string_view domain, - Attrs key, + Key key, Store & store) = 0; /** @@ -83,8 +87,7 @@ struct Cache * has exceeded `settings.tarballTTL`. */ virtual std::optional lookupStorePathWithTTL( - std::string_view domain, - Attrs key, + Key key, Store & store) = 0; }; diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 2116906ad..96743cb52 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -16,17 +16,16 @@ StorePath fetchToStore( // FIXME: add an optimisation for the case where the accessor is // an FSInputAccessor pointing to a store path. - auto domain = "fetchToStore"; - std::optional cacheKey; + std::optional cacheKey; if (!filter && path.accessor->fingerprint) { - cacheKey = fetchers::Attrs{ + cacheKey = fetchers::Cache::Key{"fetchToStore", { {"name", std::string{name}}, {"fingerprint", *path.accessor->fingerprint}, {"method", std::string{method.render()}}, {"path", path.path.abs()} - }; - if (auto res = fetchers::getCache()->lookupStorePath(domain, *cacheKey, store)) { + }}; + if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { debug("store path cache hit for '%s'", path); return res->storePath; } @@ -46,7 +45,7 @@ StorePath fetchToStore( name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair); if (cacheKey && mode == FetchMode::Copy) - fetchers::getCache()->upsert(domain, *cacheKey, store, {}, storePath); + fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); return storePath; } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ae4facc67..d4ba1a91d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -456,15 +456,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { auto accessor = getAccessor(treeHash, false); - auto domain = "treeHashToNarHash"; - fetchers::Attrs cacheKey({{"treeHash", treeHash.gitRev()}}); + fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; - if (auto res = fetchers::getCache()->lookup(domain, cacheKey)) + if (auto res = fetchers::getCache()->lookup(cacheKey)) return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); auto narHash = accessor->hashPath(CanonPath::root); - fetchers::getCache()->upsert(domain, cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); return narHash; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 487144925..7aa857dfe 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -225,13 +225,11 @@ struct GitArchiveInputScheme : InputScheme auto cache = getCache(); - auto treeHashDomain = "gitRevToTreeHash"; - Attrs treeHashKey{{"rev", rev->gitRev()}}; - auto lastModifiedDomain = "gitRevToLastModified"; - Attrs lastModifiedKey{{"rev", rev->gitRev()}}; + Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; + Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; - if (auto treeHashAttrs = cache->lookup(treeHashDomain, treeHashKey)) { - if (auto lastModifiedAttrs = cache->lookup(lastModifiedDomain, lastModifiedKey)) { + if (auto treeHashAttrs = cache->lookup(treeHashKey)) { + if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); if (getTarballCache()->hasObject(treeHash)) @@ -259,8 +257,8 @@ struct GitArchiveInputScheme : InputScheme .lastModified = lastModified }; - cache->upsert(treeHashDomain, treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); - cache->upsert(lastModifiedDomain, lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); + cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); + cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); #if 0 if (upstreamTreeHash != tarballInfo.treeHash) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 285e2803c..89ef31c7e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -23,14 +23,12 @@ DownloadFileResult downloadFile( { // FIXME: check store - auto domain = "file"; - - Attrs key({ + Cache::Key key{"file", {{ {"url", url}, {"name", name}, - }); + }}}; - auto cached = getCache()->lookupStorePath(domain, key, *store); + auto cached = getCache()->lookupStorePath(key, *store); auto useCached = [&]() -> DownloadFileResult { @@ -94,9 +92,9 @@ DownloadFileResult downloadFile( /* Cache metadata for all URLs in the redirect chain. */ for (auto & url : res.urls) { - key.insert_or_assign("url", url); + key.second.insert_or_assign("url", url); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); - getCache()->upsert(domain, key, *store, infoAttrs, *storePath); + getCache()->upsert(key, *store, infoAttrs, *storePath); } return { @@ -111,12 +109,9 @@ DownloadTarballResult downloadTarball( const std::string & url, const Headers & headers) { - auto domain = "tarball"; - Attrs cacheKey{ - {"url", url}, - }; + Cache::Key cacheKey{"tarball", {{"url", url}}}; - auto cached = getCache()->lookupExpired(domain, cacheKey); + auto cached = getCache()->lookupExpired(cacheKey); auto attrsToResult = [&](const Attrs & infoAttrs) { @@ -175,8 +170,8 @@ DownloadTarballResult downloadTarball( /* Insert a cache entry for every URL in the redirect chain. */ for (auto & url : res->urls) { - cacheKey.insert_or_assign("url", url); - getCache()->upsert(domain, cacheKey, infoAttrs); + cacheKey.second.insert_or_assign("url", url); + getCache()->upsert(cacheKey, infoAttrs); } // FIXME: add a cache entry for immutableUrl? That could allow diff --git a/src/libfetchers/unix/git.cc b/src/libfetchers/unix/git.cc index bb1bff7ab..1d7c719a6 100644 --- a/src/libfetchers/unix/git.cc +++ b/src/libfetchers/unix/git.cc @@ -427,36 +427,34 @@ struct GitInputScheme : InputScheme uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - auto domain = "gitLastModified"; - Attrs key{{"rev", rev.gitRev()}}; + Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; auto cache = getCache(); - if (auto res = cache->lookup(domain, key)) + if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev); - cache->upsert(domain, key, {{"lastModified", lastModified}}); + cache->upsert(key, {{"lastModified", lastModified}}); return lastModified; } uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - auto domain = "gitRevCount"; - Attrs key{{"rev", rev.gitRev()}}; + Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; auto cache = getCache(); - if (auto revCountAttrs = cache->lookup(domain, key)) + if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); - cache->upsert(domain, key, Attrs{{"revCount", revCount}}); + cache->upsert(key, Attrs{{"revCount", revCount}}); return revCount; } diff --git a/src/libfetchers/unix/mercurial.cc b/src/libfetchers/unix/mercurial.cc index 42757f2db..0dd1acf45 100644 --- a/src/libfetchers/unix/mercurial.cc +++ b/src/libfetchers/unix/mercurial.cc @@ -224,17 +224,16 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); - auto revInfoDomain = "hgRev"; auto revInfoKey = [&](const Hash & rev) { if (rev.algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.to_string(HashFormat::Base16, true)); - return Attrs{ + return Cache::Key{"hgRev", { {"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()} - }; + }}; }; auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath @@ -246,20 +245,19 @@ struct MercurialInputScheme : InputScheme }; /* Check the cache for the most recent rev for this URL/ref. */ - auto refToRevDomain = "hgRefToRev"; - Attrs refToRevKey{ + Cache::Key refToRevKey{"hgRefToRev", { {"url", actualUrl}, {"ref", *input.getRef()} - }; + }}; if (!input.getRev()) { - if (auto res = getCache()->lookupWithTTL(refToRevDomain, refToRevKey)) + if (auto res = getCache()->lookupWithTTL(refToRevKey)) input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); } /* If we have a rev, check if we have a cached store path. */ if (auto rev = input.getRev()) { - if (auto res = getCache()->lookupStorePath(revInfoDomain, revInfoKey(*rev), *store)) + if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) return makeResult(res->value, res->storePath); } @@ -309,7 +307,7 @@ struct MercurialInputScheme : InputScheme /* Now that we have the rev, check the cache again for a cached store path. */ - if (auto res = getCache()->lookupStorePath(revInfoDomain, revInfoKey(rev), *store)) + if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) return makeResult(res->value, res->storePath); Path tmpDir = createTempDir(); @@ -327,9 +325,9 @@ struct MercurialInputScheme : InputScheme }); if (!origRev) - getCache()->upsert(refToRevDomain, refToRevKey, {{"rev", rev.gitRev()}}); + getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); - getCache()->upsert(revInfoDomain, revInfoKey(rev), *store, infoAttrs, storePath); + getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); return makeResult(infoAttrs, std::move(storePath)); }