max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2025-01-31 15:36:47 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Combine the domain and key arguments into a single value for convenience

Browse source
This commit is contained in:
Eelco Dolstra 2024-04-10 21:39:40 +02:00
parent aad11f4496
commit cceae30aaf
8 changed files with 84 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
src/libfetchers/cache.cc
View file

@ -51,57 +51,53 @@ struct CacheImpl : Cache
} }
void upsert( void upsert(
std::string_view domain, const Key & key,
const Attrs & key,
const Attrs & value) override const Attrs & value) override
{ {
_state.lock()->upsert.use() _state.lock()->upsert.use()
(domain) (key.first)
(attrsToJSON(key).dump()) (attrsToJSON(key.second).dump())
(attrsToJSON(value).dump()) (attrsToJSON(value).dump())
(time(0)).exec(); (time(0)).exec();
} }
std::optional<Attrs> lookup( std::optional<Attrs> lookup(
std::string_view domain, const Key & key) override
const Attrs & key) override
{ {
if (auto res = lookupExpired(domain, key)) if (auto res = lookupExpired(key))
return std::move(res->value); return std::move(res->value);
return {}; return {};
} }
std::optional<Attrs> lookupWithTTL( std::optional<Attrs> lookupWithTTL(
std::string_view domain, const Key & key) override
const Attrs & key) override
{ {
if (auto res = lookupExpired(domain, key)) { if (auto res = lookupExpired(key)) {
if (!res->expired) if (!res->expired)
return std::move(res->value); return std::move(res->value);
debug("ignoring expired cache entry '%s:%s'", debug("ignoring expired cache entry '%s:%s'",
domain, attrsToJSON(key).dump()); key.first, attrsToJSON(key.second).dump());
} }
return {}; return {};
} }
std::optional<Result> lookupExpired( std::optional<Result> lookupExpired(
std::string_view domain, const Key & key) override
const Attrs & key) override
{ {
auto state(_state.lock()); auto state(_state.lock());
auto keyJSON = attrsToJSON(key).dump(); auto keyJSON = attrsToJSON(key.second).dump();
auto stmt(state->lookup.use()(domain)(keyJSON)); auto stmt(state->lookup.use()(key.first)(keyJSON));
if (!stmt.next()) { if (!stmt.next()) {
debug("did not find cache entry for '%s:%s'", domain, keyJSON); debug("did not find cache entry for '%s:%s'", key.first, keyJSON);
return {}; return {};
} }
auto valueJSON = stmt.getStr(0); auto valueJSON = stmt.getStr(0);
auto timestamp = stmt.getInt(1); auto timestamp = stmt.getInt(1);
debug("using cache entry '%s:%s' -> '%s'", domain, keyJSON, valueJSON); debug("using cache entry '%s:%s' -> '%s'", key.first, keyJSON, valueJSON);
return Result { return Result {
.expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0), .expired = settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0),
@ -110,29 +106,27 @@ struct CacheImpl : Cache
} }
void upsert( void upsert(
std::string_view domain, Key key,
Attrs key,
Store & store, Store & store,
Attrs value, Attrs value,
const StorePath & storePath) const StorePath & storePath)
{ {
/* Add the store prefix to the cache key to handle multiple /* Add the store prefix to the cache key to handle multiple
store prefixes. */ store prefixes. */
key.insert_or_assign("store", store.storeDir); key.second.insert_or_assign("store", store.storeDir);
value.insert_or_assign("storePath", (std::string) storePath.to_string()); value.insert_or_assign("storePath", (std::string) storePath.to_string());
upsert(domain, key, value); upsert(key, value);
} }
std::optional<ResultWithStorePath> lookupStorePath( std::optional<ResultWithStorePath> lookupStorePath(
std::string_view domain, Key key,
Attrs key,
Store & store) override Store & store) override
{ {
key.insert_or_assign("store", store.storeDir); key.second.insert_or_assign("store", store.storeDir);
auto res = lookupExpired(domain, key); auto res = lookupExpired(key);
if (!res) return std::nullopt; if (!res) return std::nullopt;
auto storePathS = getStrAttr(res->value, "storePath"); auto storePathS = getStrAttr(res->value, "storePath");
@ -143,14 +137,16 @@ struct CacheImpl : Cache
store.addTempRoot(res2.storePath); store.addTempRoot(res2.storePath);
if (!store.isValidPath(res2.storePath)) { if (!store.isValidPath(res2.storePath)) {
// FIXME: we could try to substitute 'storePath'. // FIXME: we could try to substitute 'storePath'.
debug("ignoring disappeared cache entry '%s' -> '%s'", debug("ignoring disappeared cache entry '%s:%s' -> '%s'",
attrsToJSON(key).dump(), key.first,
attrsToJSON(key.second).dump(),
store.printStorePath(res2.storePath)); store.printStorePath(res2.storePath));
return std::nullopt; return std::nullopt;
} }
debug("using cache entry '%s' -> '%s', '%s'", debug("using cache entry '%s:%s' -> '%s', '%s'",
attrsToJSON(key).dump(), key.first,
attrsToJSON(key.second).dump(),
attrsToJSON(res2.value).dump(), attrsToJSON(res2.value).dump(),
store.printStorePath(res2.storePath)); store.printStorePath(res2.storePath));
@ -158,11 +154,10 @@ struct CacheImpl : Cache
} }
std::optional<ResultWithStorePath> lookupStorePathWithTTL( std::optional<ResultWithStorePath> lookupStorePathWithTTL(
std::string_view domain, Key key,
Attrs key,
Store & store) override Store & store) override
{ {
auto res = lookupStorePath(domain, std::move(key), store); auto res = lookupStorePath(std::move(key), store);
return res && !res->expired ? res : std::nullopt; return res && !res->expired ? res : std::nullopt;
} }
}; };

35
src/libfetchers/cache.hh
View file

@ -15,28 +15,35 @@ struct Cache
virtual ~Cache() { } virtual ~Cache() { }
/** /**
* Add a value to the cache. The cache is an arbitrary mapping of * A domain is a partition of the key/value cache for a particular
* Attrs to Attrs. * purpose, e.g. "Git revision to revcount".
*/
using Domain = std::string_view;
/**
* A cache key is a domain and an arbitrary set of attributes.
*/
using Key = std::pair<Domain, Attrs>;
/**
* Add a key/value pair to the cache.
*/ */
virtual void upsert( virtual void upsert(
std::string_view domain, const Key & key,
const Attrs & key,
const Attrs & value) = 0; const Attrs & value) = 0;
/** /**
* Look up a key with infinite TTL. * Look up a key with infinite TTL.
*/ */
virtual std::optional<Attrs> lookup( virtual std::optional<Attrs> lookup(
std::string_view domain, const Key & key) = 0;
const Attrs & key) = 0;
/** /**
* Look up a key. Return nothing if its TTL has exceeded * Look up a key. Return nothing if its TTL has exceeded
* `settings.tarballTTL`. * `settings.tarballTTL`.
*/ */
virtual std::optional<Attrs> lookupWithTTL( virtual std::optional<Attrs> lookupWithTTL(
std::string_view domain, const Key & key) = 0;
const Attrs & key) = 0;
struct Result struct Result
{ {
@ -49,8 +56,7 @@ struct Cache
* exceeded `settings.tarballTTL`. * exceeded `settings.tarballTTL`.
*/ */
virtual std::optional<Result> lookupExpired( virtual std::optional<Result> lookupExpired(
std::string_view domain, const Key & key) = 0;
const Attrs & key) = 0;
/** /**
* Insert a cache entry that has a store path associated with * Insert a cache entry that has a store path associated with
@ -58,8 +64,7 @@ struct Cache
* associated store path is invalid. * associated store path is invalid.
*/ */
virtual void upsert( virtual void upsert(
std::string_view domain, Key key,
Attrs key,
Store & store, Store & store,
Attrs value, Attrs value,
const StorePath & storePath) = 0; const StorePath & storePath) = 0;
@ -74,8 +79,7 @@ struct Cache
* be valid, but it may be expired. * be valid, but it may be expired.
*/ */
virtual std::optional<ResultWithStorePath> lookupStorePath( virtual std::optional<ResultWithStorePath> lookupStorePath(
std::string_view domain, Key key,
Attrs key,
Store & store) = 0; Store & store) = 0;
/** /**
@ -83,8 +87,7 @@ struct Cache
* has exceeded `settings.tarballTTL`. * has exceeded `settings.tarballTTL`.
*/ */
virtual std::optional<ResultWithStorePath> lookupStorePathWithTTL( virtual std::optional<ResultWithStorePath> lookupStorePathWithTTL(
std::string_view domain, Key key,
Attrs key,
Store & store) = 0; Store & store) = 0;
}; };

11
src/libfetchers/fetch-to-store.cc
View file

@ -16,17 +16,16 @@ StorePath fetchToStore(
// FIXME: add an optimisation for the case where the accessor is // FIXME: add an optimisation for the case where the accessor is
// an FSInputAccessor pointing to a store path. // an FSInputAccessor pointing to a store path.
auto domain = "fetchToStore"; std::optional<fetchers::Cache::Key> cacheKey;
std::optional<fetchers::Attrs> cacheKey;
if (!filter && path.accessor->fingerprint) { if (!filter && path.accessor->fingerprint) {
cacheKey = fetchers::Attrs{ cacheKey = fetchers::Cache::Key{"fetchToStore", {
{"name", std::string{name}}, {"name", std::string{name}},
{"fingerprint", *path.accessor->fingerprint}, {"fingerprint", *path.accessor->fingerprint},
{"method", std::string{method.render()}}, {"method", std::string{method.render()}},
{"path", path.path.abs()} {"path", path.path.abs()}
}; }};
if (auto res = fetchers::getCache()->lookupStorePath(domain, *cacheKey, store)) { if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
debug("store path cache hit for '%s'", path); debug("store path cache hit for '%s'", path);
return res->storePath; return res->storePath;
} }
@ -46,7 +45,7 @@ StorePath fetchToStore(
name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair); name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair);
if (cacheKey && mode == FetchMode::Copy) if (cacheKey && mode == FetchMode::Copy)
fetchers::getCache()->upsert(domain, *cacheKey, store, {}, storePath); fetchers::getCache()->upsert(*cacheKey, store, {}, storePath);
return storePath; return storePath;
} }

7
src/libfetchers/git-utils.cc
View file

@ -456,15 +456,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
{ {
auto accessor = getAccessor(treeHash, false); auto accessor = getAccessor(treeHash, false);
auto domain = "treeHashToNarHash"; fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
fetchers::Attrs cacheKey({{"treeHash", treeHash.gitRev()}});
if (auto res = fetchers::getCache()->lookup(domain, cacheKey)) if (auto res = fetchers::getCache()->lookup(cacheKey))
return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256);
auto narHash = accessor->hashPath(CanonPath::root); auto narHash = accessor->hashPath(CanonPath::root);
fetchers::getCache()->upsert(domain, cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}}));
return narHash; return narHash;
} }

14
src/libfetchers/github.cc
View file

@ -225,13 +225,11 @@ struct GitArchiveInputScheme : InputScheme
auto cache = getCache(); auto cache = getCache();
auto treeHashDomain = "gitRevToTreeHash"; Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Attrs treeHashKey{{"rev", rev->gitRev()}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
auto lastModifiedDomain = "gitRevToLastModified";
Attrs lastModifiedKey{{"rev", rev->gitRev()}};
if (auto treeHashAttrs = cache->lookup(treeHashDomain, treeHashKey)) { if (auto treeHashAttrs = cache->lookup(treeHashKey)) {
if (auto lastModifiedAttrs = cache->lookup(lastModifiedDomain, lastModifiedKey)) { if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (getTarballCache()->hasObject(treeHash)) if (getTarballCache()->hasObject(treeHash))
@ -259,8 +257,8 @@ struct GitArchiveInputScheme : InputScheme
.lastModified = lastModified .lastModified = lastModified
}; };
cache->upsert(treeHashDomain, treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
cache->upsert(lastModifiedDomain, lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
#if 0 #if 0
if (upstreamTreeHash != tarballInfo.treeHash) if (upstreamTreeHash != tarballInfo.treeHash)

23
src/libfetchers/tarball.cc
View file

@ -23,14 +23,12 @@ DownloadFileResult downloadFile(
{ {
// FIXME: check store // FIXME: check store
auto domain = "file"; Cache::Key key{"file", {{
Attrs key({
{"url", url}, {"url", url},
{"name", name}, {"name", name},
}); }}};
auto cached = getCache()->lookupStorePath(domain, key, *store); auto cached = getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult auto useCached = [&]() -> DownloadFileResult
{ {
@ -94,9 +92,9 @@ DownloadFileResult downloadFile(
/* Cache metadata for all URLs in the redirect chain. */ /* Cache metadata for all URLs in the redirect chain. */
for (auto & url : res.urls) { for (auto & url : res.urls) {
key.insert_or_assign("url", url); key.second.insert_or_assign("url", url);
infoAttrs.insert_or_assign("url", *res.urls.rbegin()); infoAttrs.insert_or_assign("url", *res.urls.rbegin());
getCache()->upsert(domain, key, *store, infoAttrs, *storePath); getCache()->upsert(key, *store, infoAttrs, *storePath);
} }
return { return {
@ -111,12 +109,9 @@ DownloadTarballResult downloadTarball(
const std::string & url, const std::string & url,
const Headers & headers) const Headers & headers)
{ {
auto domain = "tarball"; Cache::Key cacheKey{"tarball", {{"url", url}}};
Attrs cacheKey{
{"url", url},
};
auto cached = getCache()->lookupExpired(domain, cacheKey); auto cached = getCache()->lookupExpired(cacheKey);
auto attrsToResult = [&](const Attrs & infoAttrs) auto attrsToResult = [&](const Attrs & infoAttrs)
{ {
@ -175,8 +170,8 @@ DownloadTarballResult downloadTarball(
/* Insert a cache entry for every URL in the redirect chain. */ /* Insert a cache entry for every URL in the redirect chain. */
for (auto & url : res->urls) { for (auto & url : res->urls) {
cacheKey.insert_or_assign("url", url); cacheKey.second.insert_or_assign("url", url);
getCache()->upsert(domain, cacheKey, infoAttrs); getCache()->upsert(cacheKey, infoAttrs);
} }
// FIXME: add a cache entry for immutableUrl? That could allow // FIXME: add a cache entry for immutableUrl? That could allow

14
src/libfetchers/unix/git.cc
View file

@ -427,36 +427,34 @@ struct GitInputScheme : InputScheme
uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
{ {
auto domain = "gitLastModified"; Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}};
Attrs key{{"rev", rev.gitRev()}};
auto cache = getCache(); auto cache = getCache();
if (auto res = cache->lookup(domain, key)) if (auto res = cache->lookup(key))
return getIntAttr(*res, "lastModified"); return getIntAttr(*res, "lastModified");
auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev); auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev);
cache->upsert(domain, key, {{"lastModified", lastModified}}); cache->upsert(key, {{"lastModified", lastModified}});
return lastModified; return lastModified;
} }
uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
{ {
auto domain = "gitRevCount"; Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}};
Attrs key{{"rev", rev.gitRev()}};
auto cache = getCache(); auto cache = getCache();
if (auto revCountAttrs = cache->lookup(domain, key)) if (auto revCountAttrs = cache->lookup(key))
return getIntAttr(*revCountAttrs, "revCount"); return getIntAttr(*revCountAttrs, "revCount");
Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev);
cache->upsert(domain, key, Attrs{{"revCount", revCount}}); cache->upsert(key, Attrs{{"revCount", revCount}});
return revCount; return revCount;
} }

20
src/libfetchers/unix/mercurial.cc
View file

@ -224,17 +224,16 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto revInfoDomain = "hgRev";
auto revInfoKey = [&](const Hash & rev) auto revInfoKey = [&](const Hash & rev)
{ {
if (rev.algo != HashAlgorithm::SHA1) if (rev.algo != HashAlgorithm::SHA1)
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.to_string(HashFormat::Base16, true)); throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", rev.to_string(HashFormat::Base16, true));
return Attrs{ return Cache::Key{"hgRev", {
{"store", store->storeDir}, {"store", store->storeDir},
{"name", name}, {"name", name},
{"rev", input.getRev()->gitRev()} {"rev", input.getRev()->gitRev()}
}; }};
}; };
auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath
@ -246,20 +245,19 @@ struct MercurialInputScheme : InputScheme
}; };
/* Check the cache for the most recent rev for this URL/ref. */ /* Check the cache for the most recent rev for this URL/ref. */
auto refToRevDomain = "hgRefToRev"; Cache::Key refToRevKey{"hgRefToRev", {
Attrs refToRevKey{
{"url", actualUrl}, {"url", actualUrl},
{"ref", *input.getRef()} {"ref", *input.getRef()}
}; }};
if (!input.getRev()) { if (!input.getRev()) {
if (auto res = getCache()->lookupWithTTL(refToRevDomain, refToRevKey)) if (auto res = getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
} }
/* If we have a rev, check if we have a cached store path. */ /* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
if (auto res = getCache()->lookupStorePath(revInfoDomain, revInfoKey(*rev), *store)) if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
} }
@ -309,7 +307,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a /* Now that we have the rev, check the cache again for a
cached store path. */ cached store path. */
if (auto res = getCache()->lookupStorePath(revInfoDomain, revInfoKey(rev), *store)) if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@ -327,9 +325,9 @@ struct MercurialInputScheme : InputScheme
}); });
if (!origRev) if (!origRev)
getCache()->upsert(refToRevDomain, refToRevKey, {{"rev", rev.gitRev()}}); getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
getCache()->upsert(revInfoDomain, revInfoKey(rev), *store, infoAttrs, storePath); getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath)); return makeResult(infoAttrs, std::move(storePath));
} }