mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-11 00:36:20 +02:00
2df9cbeb47
E.g. $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m4.139s $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nars \ /nix/store/b0w2hafndl09h64fhb86kw6bmhbmnpm1-blender-2.79/share/icons/hicolor/scalable/apps/blender.svg > /dev/null real 0m0.024s (Before, the second call took ~0.220s.) This will use a NAR listing in /tmp/nars/b0w2hafndl09h64fhb86kw6bmhbmnpm1.ls containing all metadata, including the offsets of regular files inside the NAR. Thus, we don't need to read the entire NAR. (We do read the entire listing, but that's generally pretty small. We could use a SQLite DB by borrowing some more code from nixos-channel-scripts/file-cache.hh.) This is primarily useful when Hydra is serving files from an S3 binary cache, in particular when you have giant NARs. E.g. we had some 12 GiB NARs, so accessing individuals files was pretty slow.
129 lines
3.5 KiB
C++
129 lines
3.5 KiB
C++
#include "remote-fs-accessor.hh"
|
|
#include "nar-accessor.hh"
|
|
#include "json.hh"
|
|
|
|
#include <sys/types.h>
|
|
#include <sys/stat.h>
|
|
#include <fcntl.h>
|
|
|
|
namespace nix {
|
|
|
|
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
|
|
: store(store)
|
|
, cacheDir(cacheDir)
|
|
{
|
|
if (cacheDir != "")
|
|
createDirs(cacheDir);
|
|
}
|
|
|
|
Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
|
|
{
|
|
assert(cacheDir != "");
|
|
return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext);
|
|
}
|
|
|
|
void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
|
|
ref<FSAccessor> narAccessor)
|
|
{
|
|
nars.emplace(storePath, narAccessor);
|
|
|
|
if (cacheDir != "") {
|
|
try {
|
|
std::ostringstream str;
|
|
JSONPlaceholder jsonRoot(str);
|
|
listNar(jsonRoot, narAccessor, "", true);
|
|
writeFile(makeCacheFile(storePath, "ls"), str.str());
|
|
|
|
/* FIXME: do this asynchronously. */
|
|
writeFile(makeCacheFile(storePath, "nar"), nar);
|
|
|
|
} catch (...) {
|
|
ignoreException();
|
|
}
|
|
}
|
|
}
|
|
|
|
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
|
{
|
|
auto path = canonPath(path_);
|
|
|
|
auto storePath = store->toStorePath(path);
|
|
std::string restPath = std::string(path, storePath.size());
|
|
|
|
if (!store->isValidPath(storePath))
|
|
throw InvalidPath(format("path '%1%' is not a valid store path") % storePath);
|
|
|
|
auto i = nars.find(storePath);
|
|
if (i != nars.end()) return {i->second, restPath};
|
|
|
|
StringSink sink;
|
|
std::string listing;
|
|
Path cacheFile;
|
|
|
|
if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
|
|
|
|
try {
|
|
listing = nix::readFile(makeCacheFile(storePath, "ls"));
|
|
|
|
auto narAccessor = makeLazyNarAccessor(listing,
|
|
[cacheFile](uint64_t offset, uint64_t length) {
|
|
|
|
AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC);
|
|
if (!fd)
|
|
throw SysError("opening NAR cache file '%s'", cacheFile);
|
|
|
|
if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset)
|
|
throw SysError("seeking in '%s'", cacheFile);
|
|
|
|
std::string buf(length, 0);
|
|
readFull(fd.get(), (unsigned char *) buf.data(), length);
|
|
|
|
return buf;
|
|
});
|
|
|
|
nars.emplace(storePath, narAccessor);
|
|
return {narAccessor, restPath};
|
|
|
|
} catch (SysError &) { }
|
|
|
|
try {
|
|
*sink.s = nix::readFile(cacheFile);
|
|
|
|
auto narAccessor = makeNarAccessor(sink.s);
|
|
nars.emplace(storePath, narAccessor);
|
|
return {narAccessor, restPath};
|
|
|
|
} catch (SysError &) { }
|
|
}
|
|
|
|
store->narFromPath(storePath, sink);
|
|
auto narAccessor = makeNarAccessor(sink.s);
|
|
addToCache(storePath, *sink.s, narAccessor);
|
|
return {narAccessor, restPath};
|
|
}
|
|
|
|
FSAccessor::Stat RemoteFSAccessor::stat(const Path & path)
|
|
{
|
|
auto res = fetch(path);
|
|
return res.first->stat(res.second);
|
|
}
|
|
|
|
StringSet RemoteFSAccessor::readDirectory(const Path & path)
|
|
{
|
|
auto res = fetch(path);
|
|
return res.first->readDirectory(res.second);
|
|
}
|
|
|
|
std::string RemoteFSAccessor::readFile(const Path & path)
|
|
{
|
|
auto res = fetch(path);
|
|
return res.first->readFile(res.second);
|
|
}
|
|
|
|
std::string RemoteFSAccessor::readLink(const Path & path)
|
|
{
|
|
auto res = fetch(path);
|
|
return res.first->readLink(res.second);
|
|
}
|
|
|
|
}
|