mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-10 08:16:15 +02:00
Add NAR / Store accessor abstraction
This is primary to allow hydra-queue-runner to extract files like "nix-support/hydra-build-products" from NARs in binary caches.
This commit is contained in:
parent
152b1d6bf9
commit
1042c10fd0
11 changed files with 352 additions and 13 deletions
|
@ -1,12 +1,13 @@
|
|||
#include "binary-cache-store.hh"
|
||||
#include "sync.hh"
|
||||
|
||||
#include "archive.hh"
|
||||
#include "binary-cache-store.hh"
|
||||
#include "compression.hh"
|
||||
#include "derivations.hh"
|
||||
#include "fs-accessor.hh"
|
||||
#include "globals.hh"
|
||||
#include "nar-info.hh"
|
||||
#include "sync.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "nar-accessor.hh"
|
||||
|
||||
#include <chrono>
|
||||
|
||||
|
@ -122,7 +123,8 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
|
|||
|
||||
auto narInfoFile = narInfoFileFor(storePath);
|
||||
auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
|
||||
assert(narInfo->path == storePath);
|
||||
if (narInfo->path != storePath)
|
||||
throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
|
||||
|
||||
stats.narInfoRead++;
|
||||
|
||||
|
@ -142,6 +144,9 @@ NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
|
|||
|
||||
bool BinaryCacheStore::isValidPath(const Path & storePath)
|
||||
{
|
||||
// FIXME: this only checks whether a .narinfo with a matching hash
|
||||
// part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even
|
||||
// though they shouldn't. Not easily fixed.
|
||||
return fileExists(narInfoFileFor(storePath));
|
||||
}
|
||||
|
||||
|
@ -344,4 +349,71 @@ void BinaryCacheStore::ensurePath(const Path & path)
|
|||
buildPaths({path});
|
||||
}
|
||||
|
||||
/* Given requests for a path /nix/store/<x>/<y>, this accessor will
|
||||
first download the NAR for /nix/store/<x> from the binary cache,
|
||||
build a NAR accessor for that NAR, and use that to access <y>. */
|
||||
struct BinaryCacheStoreAccessor : public FSAccessor
|
||||
{
|
||||
ref<BinaryCacheStore> store;
|
||||
|
||||
std::map<Path, ref<FSAccessor>> nars;
|
||||
|
||||
BinaryCacheStoreAccessor(ref<BinaryCacheStore> store)
|
||||
: store(store)
|
||||
{
|
||||
}
|
||||
|
||||
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_)
|
||||
{
|
||||
auto path = canonPath(path_);
|
||||
|
||||
auto storePath = toStorePath(path);
|
||||
std::string restPath = std::string(path, storePath.size());
|
||||
|
||||
if (!store->isValidPath(storePath))
|
||||
throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
|
||||
|
||||
auto i = nars.find(storePath);
|
||||
if (i != nars.end()) return {i->second, restPath};
|
||||
|
||||
StringSink sink;
|
||||
store->exportPath(storePath, false, sink);
|
||||
|
||||
// FIXME: gratuitous string copying.
|
||||
auto accessor = makeNarAccessor(make_ref<std::string>(sink.s));
|
||||
nars.emplace(storePath, accessor);
|
||||
return {accessor, restPath};
|
||||
}
|
||||
|
||||
Stat stat(const Path & path) override
|
||||
{
|
||||
auto res = fetch(path);
|
||||
return res.first->stat(res.second);
|
||||
}
|
||||
|
||||
StringSet readDirectory(const Path & path) override
|
||||
{
|
||||
auto res = fetch(path);
|
||||
return res.first->readDirectory(res.second);
|
||||
}
|
||||
|
||||
std::string readFile(const Path & path) override
|
||||
{
|
||||
auto res = fetch(path);
|
||||
return res.first->readFile(res.second);
|
||||
}
|
||||
|
||||
std::string readLink(const Path & path) override
|
||||
{
|
||||
auto res = fetch(path);
|
||||
return res.first->readLink(res.second);
|
||||
}
|
||||
};
|
||||
|
||||
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>(
|
||||
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -125,8 +125,7 @@ public:
|
|||
Path addTextToStore(const string & name, const string & s,
|
||||
const PathSet & references, bool repair = false) override;
|
||||
|
||||
void exportPath(const Path & path, bool sign,
|
||||
Sink & sink) override;
|
||||
void exportPath(const Path & path, bool sign, Sink & sink) override;
|
||||
|
||||
Paths importPaths(bool requireSignature, Source & source) override;
|
||||
|
||||
|
@ -167,6 +166,8 @@ public:
|
|||
bool verifyStore(bool checkContents, bool repair) override
|
||||
{ return true; }
|
||||
|
||||
ref<FSAccessor> getFSAccessor() override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
|
30
src/libstore/fs-accessor.hh
Normal file
30
src/libstore/fs-accessor.hh
Normal file
|
@ -0,0 +1,30 @@
|
|||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* An abstract class for accessing a filesystem-like structure, such
|
||||
as a (possibly remote) Nix store or the contents of a NAR file. */
|
||||
class FSAccessor
|
||||
{
|
||||
public:
|
||||
enum Type { tMissing, tRegular, tSymlink, tDirectory };
|
||||
|
||||
struct Stat
|
||||
{
|
||||
Type type;
|
||||
uint64_t fileSize; // regular files only
|
||||
bool isExecutable; // regular files only
|
||||
};
|
||||
|
||||
virtual Stat stat(const Path & path) = 0;
|
||||
|
||||
virtual StringSet readDirectory(const Path & path) = 0;
|
||||
|
||||
virtual std::string readFile(const Path & path) = 0;
|
||||
|
||||
virtual std::string readLink(const Path & path) = 0;
|
||||
};
|
||||
|
||||
}
|
71
src/libstore/local-fs-store.cc
Normal file
71
src/libstore/local-fs-store.cc
Normal file
|
@ -0,0 +1,71 @@
|
|||
#include "fs-accessor.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct LocalStoreAccessor : public FSAccessor
|
||||
{
|
||||
ref<Store> store;
|
||||
|
||||
LocalStoreAccessor(ref<Store> store) : store(store) { }
|
||||
|
||||
void assertStore(const Path & path)
|
||||
{
|
||||
Path storePath = toStorePath(path);
|
||||
if (!store->isValidPath(storePath))
|
||||
throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
|
||||
}
|
||||
|
||||
FSAccessor::Stat stat(const Path & path) override
|
||||
{
|
||||
assertStore(path);
|
||||
|
||||
struct stat st;
|
||||
if (lstat(path.c_str(), &st)) {
|
||||
if (errno == ENOENT) return {Type::tMissing, 0, false};
|
||||
throw SysError(format("getting status of ‘%1%’") % path);
|
||||
}
|
||||
|
||||
if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode))
|
||||
throw Error(format("file ‘%1%’ has unsupported type") % path);
|
||||
|
||||
return {
|
||||
S_ISREG(st.st_mode) ? Type::tRegular :
|
||||
S_ISLNK(st.st_mode) ? Type::tSymlink :
|
||||
Type::tDirectory,
|
||||
S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0,
|
||||
S_ISREG(st.st_mode) && st.st_mode & S_IXUSR};
|
||||
}
|
||||
|
||||
StringSet readDirectory(const Path & path) override
|
||||
{
|
||||
assertStore(path);
|
||||
|
||||
auto entries = nix::readDirectory(path);
|
||||
|
||||
StringSet res;
|
||||
for (auto & entry : entries)
|
||||
res.insert(entry.name);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string readFile(const Path & path) override
|
||||
{
|
||||
assertStore(path);
|
||||
return nix::readFile(path);
|
||||
}
|
||||
|
||||
std::string readLink(const Path & path) override
|
||||
{
|
||||
assertStore(path);
|
||||
return nix::readLink(path);
|
||||
}
|
||||
};
|
||||
|
||||
ref<FSAccessor> LocalFSStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this()));
|
||||
}
|
||||
|
||||
}
|
|
@ -80,7 +80,7 @@ struct SQLiteStmt
|
|||
};
|
||||
|
||||
|
||||
class LocalStore : public Store
|
||||
class LocalStore : public LocalFSStore
|
||||
{
|
||||
private:
|
||||
typedef std::map<Path, RunningSubstituter> RunningSubstituters;
|
||||
|
@ -170,14 +170,11 @@ public:
|
|||
files with the same contents. */
|
||||
void optimiseStore(OptimiseStats & stats);
|
||||
|
||||
/* Generic variant of the above method. */
|
||||
void optimiseStore() override;
|
||||
|
||||
/* Optimise a single store path. */
|
||||
void optimisePath(const Path & path);
|
||||
|
||||
/* Check the integrity of the Nix store. Returns true if errors
|
||||
remain. */
|
||||
bool verifyStore(bool checkContents, bool repair) override;
|
||||
|
||||
/* Register the validity of a path, i.e., that `path' exists, that
|
||||
|
|
142
src/libstore/nar-accessor.cc
Normal file
142
src/libstore/nar-accessor.cc
Normal file
|
@ -0,0 +1,142 @@
|
|||
#include "nar-accessor.hh"
|
||||
#include "archive.hh"
|
||||
|
||||
#include <map>
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct NarMember
|
||||
{
|
||||
FSAccessor::Type type;
|
||||
|
||||
bool isExecutable;
|
||||
|
||||
/* If this is a regular file, position of the contents of this
|
||||
file in the NAR. */
|
||||
size_t start, size;
|
||||
|
||||
std::string target;
|
||||
};
|
||||
|
||||
struct NarIndexer : ParseSink, StringSource
|
||||
{
|
||||
// FIXME: should store this as a tree. Now we're vulnerable to
|
||||
// O(nm) memory consumption (e.g. for x_0/.../x_n/{y_0..y_m}).
|
||||
typedef std::map<Path, NarMember> Members;
|
||||
Members members;
|
||||
|
||||
Path currentPath;
|
||||
std::string currentStart;
|
||||
bool isExec;
|
||||
|
||||
NarIndexer(const std::string & nar) : StringSource(nar)
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const Path & path)
|
||||
{
|
||||
members.emplace(path,
|
||||
NarMember{FSAccessor::Type::tDirectory, false, 0, 0});
|
||||
}
|
||||
|
||||
void createRegularFile(const Path & path) override
|
||||
{
|
||||
currentPath = path;
|
||||
}
|
||||
|
||||
void isExecutable()
|
||||
{
|
||||
isExec = true;
|
||||
}
|
||||
|
||||
void preallocateContents(unsigned long long size) override
|
||||
{
|
||||
assert(currentPath != "");
|
||||
currentStart = string(s, pos, 16);
|
||||
members.emplace(currentPath,
|
||||
NarMember{FSAccessor::Type::tRegular, isExec, pos, size});
|
||||
}
|
||||
|
||||
void receiveContents(unsigned char * data, unsigned int len) override
|
||||
{
|
||||
// Sanity check
|
||||
if (!currentStart.empty()) {
|
||||
assert(len < 16 || currentStart == string((char *) data, 16));
|
||||
currentStart.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void createSymlink(const Path & path, const string & target) override
|
||||
{
|
||||
members.emplace(path,
|
||||
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
||||
}
|
||||
|
||||
Members::iterator find(const Path & path)
|
||||
{
|
||||
auto i = members.find(path);
|
||||
if (i == members.end())
|
||||
throw Error(format("NAR file does not contain path ‘%1%’") % path);
|
||||
return i;
|
||||
}
|
||||
};
|
||||
|
||||
struct NarAccessor : public FSAccessor
|
||||
{
|
||||
ref<const std::string> nar;
|
||||
NarIndexer indexer;
|
||||
|
||||
NarAccessor(ref<const std::string> nar) : nar(nar), indexer(*nar)
|
||||
{
|
||||
parseDump(indexer, indexer);
|
||||
}
|
||||
|
||||
Stat stat(const Path & path) override
|
||||
{
|
||||
auto i = indexer.members.find(path);
|
||||
if (i == indexer.members.end())
|
||||
return {FSAccessor::Type::tMissing, 0, false};
|
||||
return {i->second.type, i->second.size, i->second.isExecutable};
|
||||
}
|
||||
|
||||
StringSet readDirectory(const Path & path) override
|
||||
{
|
||||
auto i = indexer.find(path);
|
||||
|
||||
if (i->second.type != FSAccessor::Type::tDirectory)
|
||||
throw Error(format("path ‘%1%’ inside NAR file is not a directory") % path);
|
||||
|
||||
++i;
|
||||
StringSet res;
|
||||
while (i != indexer.members.end() && isInDir(i->first, path)) {
|
||||
// FIXME: really bad performance.
|
||||
if (i->first.find('/', path.size() + 1) == std::string::npos)
|
||||
res.insert(std::string(i->first, path.size() + 1));
|
||||
++i;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string readFile(const Path & path) override
|
||||
{
|
||||
auto i = indexer.find(path);
|
||||
if (i->second.type != FSAccessor::Type::tRegular)
|
||||
throw Error(format("path ‘%1%’ inside NAR file is not a regular file") % path);
|
||||
return std::string(*nar, i->second.start, i->second.size);
|
||||
}
|
||||
|
||||
std::string readLink(const Path & path) override
|
||||
{
|
||||
auto i = indexer.find(path);
|
||||
if (i->second.type != FSAccessor::Type::tSymlink)
|
||||
throw Error(format("path ‘%1%’ inside NAR file is not a symlink") % path);
|
||||
return i->second.target;
|
||||
}
|
||||
};
|
||||
|
||||
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
|
||||
{
|
||||
return make_ref<NarAccessor>(nar);
|
||||
}
|
||||
|
||||
}
|
11
src/libstore/nar-accessor.hh
Normal file
11
src/libstore/nar-accessor.hh
Normal file
|
@ -0,0 +1,11 @@
|
|||
#pragma once
|
||||
|
||||
#include "fs-accessor.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
/* Return an object that provides access to the contents of a NAR
|
||||
file. */
|
||||
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
|
||||
|
||||
}
|
|
@ -16,7 +16,9 @@ struct FdSource;
|
|||
template<typename T> class Pool;
|
||||
|
||||
|
||||
class RemoteStore : public Store
|
||||
/* FIXME: RemoteStore is a misnomer - should be something like
|
||||
DaemonStore. */
|
||||
class RemoteStore : public LocalFSStore
|
||||
{
|
||||
public:
|
||||
|
||||
|
|
|
@ -332,7 +332,9 @@ ref<Store> openStoreAt(const std::string & uri)
|
|||
|
||||
enum { mDaemon, mLocal, mAuto } mode;
|
||||
|
||||
mode = uri == "daemon" ? mDaemon : mAuto;
|
||||
mode =
|
||||
uri == "daemon" ? mDaemon :
|
||||
uri == "local" ? mLocal : mAuto;
|
||||
|
||||
if (mode == mAuto) {
|
||||
if (LocalStore::haveWriteAccess())
|
||||
|
|
|
@ -140,9 +140,10 @@ struct BuildResult
|
|||
|
||||
struct BasicDerivation;
|
||||
struct Derivation;
|
||||
struct FSAccessor;
|
||||
|
||||
|
||||
class Store
|
||||
class Store : public std::enable_shared_from_this<Store>
|
||||
{
|
||||
public:
|
||||
|
||||
|
@ -314,6 +315,9 @@ public:
|
|||
remain. */
|
||||
virtual bool verifyStore(bool checkContents, bool repair) = 0;
|
||||
|
||||
/* Return an object to access files in the Nix store. */
|
||||
virtual ref<FSAccessor> getFSAccessor() = 0;
|
||||
|
||||
/* Utility functions. */
|
||||
|
||||
/* Read a derivation, after ensuring its existence through
|
||||
|
@ -345,6 +349,12 @@ public:
|
|||
};
|
||||
|
||||
|
||||
class LocalFSStore : public Store
|
||||
{
|
||||
ref<FSAccessor> getFSAccessor() override;
|
||||
};
|
||||
|
||||
|
||||
/* !!! These should be part of the store API, I guess. */
|
||||
|
||||
/* Throw an exception if `path' is not directly in the Nix store. */
|
||||
|
|
|
@ -57,6 +57,7 @@ void dumpPath(const Path & path, Sink & sink,
|
|||
|
||||
void dumpString(const std::string & s, Sink & sink);
|
||||
|
||||
/* FIXME: fix this API, it sucks. */
|
||||
struct ParseSink
|
||||
{
|
||||
virtual void createDirectory(const Path & path) { };
|
||||
|
|
Loading…
Reference in a new issue