mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-10 08:16:15 +02:00
importPaths(): Optionally add NARs to binary cache accessor
This enables an optimisation in hydra-queue-runner, preventing a download of a NAR it just uploaded to the cache when reading files like hydra-build-products.
This commit is contained in:
parent
00b2c05749
commit
cca4a8dc1a
10 changed files with 58 additions and 44 deletions
|
@ -186,7 +186,7 @@ void importPaths(int fd)
|
|||
PPCODE:
|
||||
try {
|
||||
FdSource source(fd);
|
||||
store()->importPaths(false, source);
|
||||
store()->importPaths(false, source, 0);
|
||||
} catch (Error & e) {
|
||||
croak("%s", e.what());
|
||||
}
|
||||
|
|
|
@ -183,7 +183,8 @@ void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink
|
|||
sink << exportMagic << storePath << res.references << res.deriver << 0;
|
||||
}
|
||||
|
||||
Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
|
||||
Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor)
|
||||
{
|
||||
assert(!requireSignature);
|
||||
Paths res;
|
||||
|
@ -191,7 +192,7 @@ Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
|
|||
unsigned long long n = readLongLong(source);
|
||||
if (n == 0) break;
|
||||
if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
|
||||
res.push_back(importPath(source));
|
||||
res.push_back(importPath(source, accessor));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
@ -215,34 +216,6 @@ struct NopSink : ParseSink
|
|||
{
|
||||
};
|
||||
|
||||
Path BinaryCacheStore::importPath(Source & source)
|
||||
{
|
||||
/* FIXME: some cut&paste of LocalStore::importPath(). */
|
||||
|
||||
/* Extract the NAR from the source. */
|
||||
TeeSource tee(source);
|
||||
NopSink sink;
|
||||
parseDump(sink, tee);
|
||||
|
||||
uint32_t magic = readInt(source);
|
||||
if (magic != exportMagic)
|
||||
throw Error("Nix archive cannot be imported; wrong format");
|
||||
|
||||
ValidPathInfo info;
|
||||
info.path = readStorePath(source);
|
||||
|
||||
info.references = readStorePaths<PathSet>(source);
|
||||
|
||||
readString(source); // deriver, don't care
|
||||
|
||||
bool haveSignature = readInt(source) == 1;
|
||||
assert(!haveSignature);
|
||||
|
||||
addToCache(info, tee.data);
|
||||
|
||||
return info.path;
|
||||
}
|
||||
|
||||
ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
|
||||
{
|
||||
return ValidPathInfo(readNarInfo(storePath));
|
||||
|
@ -416,4 +389,37 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
|||
std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this())));
|
||||
}
|
||||
|
||||
Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> accessor)
|
||||
{
|
||||
/* FIXME: some cut&paste of LocalStore::importPath(). */
|
||||
|
||||
/* Extract the NAR from the source. */
|
||||
TeeSource tee(source);
|
||||
NopSink sink;
|
||||
parseDump(sink, tee);
|
||||
|
||||
uint32_t magic = readInt(source);
|
||||
if (magic != exportMagic)
|
||||
throw Error("Nix archive cannot be imported; wrong format");
|
||||
|
||||
ValidPathInfo info;
|
||||
info.path = readStorePath(source);
|
||||
|
||||
info.references = readStorePaths<PathSet>(source);
|
||||
|
||||
readString(source); // deriver, don't care
|
||||
|
||||
bool haveSignature = readInt(source) == 1;
|
||||
assert(!haveSignature);
|
||||
|
||||
addToCache(info, tee.data);
|
||||
|
||||
auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor);
|
||||
if (accessor_)
|
||||
// FIXME: more gratuitous string copying
|
||||
accessor_->nars.emplace(info.path, makeNarAccessor(make_ref<std::string>(tee.data)));
|
||||
|
||||
return info.path;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -127,9 +127,10 @@ public:
|
|||
|
||||
void exportPath(const Path & path, bool sign, Sink & sink) override;
|
||||
|
||||
Paths importPaths(bool requireSignature, Source & source) override;
|
||||
Paths importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
Path importPath(Source & source);
|
||||
Path importPath(Source & source, std::shared_ptr<FSAccessor> accessor);
|
||||
|
||||
void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
|
||||
|
||||
|
|
|
@ -1689,7 +1689,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
|
|||
}
|
||||
|
||||
|
||||
Paths LocalStore::importPaths(bool requireSignature, Source & source)
|
||||
Paths LocalStore::importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor)
|
||||
{
|
||||
Paths res;
|
||||
while (true) {
|
||||
|
|
|
@ -147,7 +147,8 @@ public:
|
|||
void exportPath(const Path & path, bool sign,
|
||||
Sink & sink) override;
|
||||
|
||||
Paths importPaths(bool requireSignature, Source & source) override;
|
||||
Paths importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
void buildPaths(const PathSet & paths, BuildMode buildMode) override;
|
||||
|
||||
|
|
|
@ -406,7 +406,8 @@ void RemoteStore::exportPath(const Path & path, bool sign,
|
|||
}
|
||||
|
||||
|
||||
Paths RemoteStore::importPaths(bool requireSignature, Source & source)
|
||||
Paths RemoteStore::importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor)
|
||||
{
|
||||
auto conn(connections->get());
|
||||
conn->to << wopImportPaths;
|
||||
|
|
|
@ -65,7 +65,8 @@ public:
|
|||
void exportPath(const Path & path, bool sign,
|
||||
Sink & sink) override;
|
||||
|
||||
Paths importPaths(bool requireSignature, Source & source) override;
|
||||
Paths importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
|
||||
void buildPaths(const PathSet & paths, BuildMode buildMode) override;
|
||||
|
||||
|
|
|
@ -227,8 +227,11 @@ public:
|
|||
void exportPaths(const Paths & paths, bool sign, Sink & sink);
|
||||
|
||||
/* Import a sequence of NAR dumps created by exportPaths() into
|
||||
the Nix store. */
|
||||
virtual Paths importPaths(bool requireSignature, Source & source) = 0;
|
||||
the Nix store. Optionally, the contents of the NARs are
|
||||
preloaded into the specified FS accessor to speed up subsequent
|
||||
access. */
|
||||
virtual Paths importPaths(bool requireSignature, Source & source,
|
||||
std::shared_ptr<FSAccessor> accessor) = 0;
|
||||
|
||||
/* For each path, if it's a derivation, build it. Building a
|
||||
derivation means ensuring that the output paths are valid. If
|
||||
|
|
|
@ -310,7 +310,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
|
|||
case wopImportPaths: {
|
||||
startWork();
|
||||
TunnelSource source(from);
|
||||
Paths paths = store->importPaths(!trusted, source);
|
||||
Paths paths = store->importPaths(!trusted, source, 0);
|
||||
stopWork();
|
||||
to << paths;
|
||||
break;
|
||||
|
@ -322,8 +322,8 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe
|
|||
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
|
||||
mode = (BuildMode)readInt(from);
|
||||
|
||||
/* Repairing is not atomic, so disallowed for "untrusted"
|
||||
clients. */
|
||||
/* Repairing is not atomic, so disallowed for "untrusted"
|
||||
clients. */
|
||||
if (mode == bmRepair && !trusted)
|
||||
throw Error("repairing is not supported when building through the Nix daemon");
|
||||
}
|
||||
|
|
|
@ -732,7 +732,7 @@ static void opImport(Strings opFlags, Strings opArgs)
|
|||
if (!opArgs.empty()) throw UsageError("no arguments expected");
|
||||
|
||||
FdSource source(STDIN_FILENO);
|
||||
Paths paths = store->importPaths(requireSignature, source);
|
||||
Paths paths = store->importPaths(requireSignature, source, 0);
|
||||
|
||||
for (auto & i : paths)
|
||||
cout << format("%1%\n") % i << std::flush;
|
||||
|
@ -935,7 +935,7 @@ static void opServe(Strings opFlags, Strings opArgs)
|
|||
|
||||
case cmdImportPaths: {
|
||||
if (!writeAllowed) throw Error("importing paths is not allowed");
|
||||
store->importPaths(false, in);
|
||||
store->importPaths(false, in, 0);
|
||||
out << 1; // indicate success
|
||||
break;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue