Add command 'nix store copy-log'

Fixes #5222.
This commit is contained in:
Eelco Dolstra 2022-01-17 19:45:21 +01:00
parent 6448ea84ab
commit 4dda1f92aa
13 changed files with 165 additions and 60 deletions

View file

@ -7,3 +7,5 @@
set or toggle display of error traces. set or toggle display of error traces.
* New builtin function `builtins.zipAttrsWith` with same functionality * New builtin function `builtins.zipAttrsWith` with same functionality
as `lib.zipAttrsWith` from nixpkgs, but much more efficient. as `lib.zipAttrsWith` from nixpkgs, but much more efficient.
* New command `nix store copy-log` to copy build logs from one store
to another.

View file

@ -54,6 +54,36 @@ void StoreCommand::run()
run(getStore()); run(getStore());
} }
CopyCommand::CopyCommand()
{
addFlag({
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
});
}
ref<Store> CopyCommand::createStore()
{
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
ref<Store> CopyCommand::getDstStore()
{
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
return dstUri.empty() ? openStore() : openStore(dstUri);
}
EvalCommand::EvalCommand() EvalCommand::EvalCommand()
{ {
} }
@ -159,43 +189,6 @@ void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
run(store, std::move(sorted)); run(store, std::move(sorted));
} }
CopyCommand::CopyCommand()
: BuiltPathsCommand(true)
{
addFlag({
.longName = "from",
.description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
.description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
});
}
ref<Store> CopyCommand::createStore()
{
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
void CopyCommand::run(ref<Store> store)
{
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
BuiltPathsCommand::run(store);
}
void CopyCommand::run(ref<Store> srcStore, BuiltPaths && paths)
{
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
run(srcStore, dstStore, std::move(paths));
}
void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths) void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
{ {
if (storePaths.size() != 1) if (storePaths.size() != 1)

View file

@ -43,6 +43,19 @@ private:
std::shared_ptr<Store> _store; std::shared_ptr<Store> _store;
}; };
/* A command that copies something between `--from` and `--to`
stores. */
struct CopyCommand : virtual StoreCommand
{
std::string srcUri, dstUri;
CopyCommand();
ref<Store> createStore() override;
ref<Store> getDstStore();
};
struct EvalCommand : virtual StoreCommand, MixEvalArgs struct EvalCommand : virtual StoreCommand, MixEvalArgs
{ {
EvalCommand(); EvalCommand();
@ -176,23 +189,6 @@ public:
bool useDefaultInstallables() override { return !all; } bool useDefaultInstallables() override { return !all; }
}; };
/* A command that copies something between `--from` and `--to`
stores. */
struct CopyCommand : virtual BuiltPathsCommand
{
std::string srcUri, dstUri;
CopyCommand();
ref<Store> createStore() override;
void run(ref<Store> store) override;
void run(ref<Store> srcStore, BuiltPaths && paths) override;
virtual void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) = 0;
};
struct StorePathsCommand : public BuiltPathsCommand struct StorePathsCommand : public BuiltPathsCommand
{ {
StorePathsCommand(bool recursive = false); StorePathsCommand(bool recursive = false);

View file

@ -468,10 +468,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
dontCheckSigs = false; dontCheckSigs = false;
logger->startWork(); logger->startWork();
{
FramedSource source(from); FramedSource source(from);
store->addMultipleToStore(source, store->addMultipleToStore(source,
RepairFlag{repair}, RepairFlag{repair},
dontCheckSigs ? NoCheckSigs : CheckSigs); dontCheckSigs ? NoCheckSigs : CheckSigs);
}
logger->stopWork(); logger->stopWork();
break; break;
} }
@ -920,6 +922,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break; break;
} }
case wopAddBuildLog: {
StorePath path{readString(from)};
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
store->addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
break;
}
default: default:
throw Error("invalid operation %1%", op); throw Error("invalid operation %1%", op);
} }

View file

@ -9,6 +9,7 @@
#include "callback.hh" #include "callback.hh"
#include "topo-sort.hh" #include "topo-sort.hh"
#include "finally.hh" #include "finally.hh"
#include "compression.hh"
#include <iostream> #include <iostream>
#include <algorithm> #include <algorithm>
@ -1898,4 +1899,24 @@ FixedOutputHash LocalStore::hashCAPath(
}; };
} }
void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
assert(drvPath.isDerivation());
auto baseName = drvPath.to_string();
auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2));
if (pathExists(logPath)) return;
createDirs(dirOf(logPath));
auto tmpFile = fmt("%s.tmp.%d", logPath, getpid());
writeFile(tmpFile, compress("bzip2", log));
if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
}
} // namespace nix } // namespace nix

View file

@ -280,6 +280,8 @@ private:
const std::string_view pathHash const std::string_view pathHash
); );
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
friend struct LocalDerivationGoal; friend struct LocalDerivationGoal;
friend struct PathSubstitutionGoal; friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal; friend struct SubstitutionGoal;

View file

@ -908,6 +908,18 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
} }
void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log)
{
auto conn(getConnection());
conn->to << wopAddBuildLog << drvPath.to_string();
StringSource source(log);
conn.withFramedSink([&](Sink & sink) {
source.drainInto(sink);
});
readInt(conn->from);
}
void RemoteStore::connect() void RemoteStore::connect()
{ {
auto conn(getConnection()); auto conn(getConnection());

View file

@ -116,6 +116,8 @@ public:
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override; uint64_t & downloadSize, uint64_t & narSize) override;
void addBuildLog(const StorePath & drvPath, std::string_view log) override;
void connect() override; void connect() override;
unsigned int getProtocol() override; unsigned int getProtocol() override;

View file

@ -727,6 +727,9 @@ public:
virtual std::optional<std::string> getBuildLog(const StorePath & path) virtual std::optional<std::string> getBuildLog(const StorePath & path)
{ return std::nullopt; } { return std::nullopt; }
virtual void addBuildLog(const StorePath & path, std::string_view log)
{ unsupported("addBuildLog"); }
/* Hack to allow long-running processes like hydra-queue-runner to /* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */ occasionally flush their path info cache. */
void clearPathInfoCache() void clearPathInfoCache()

View file

@ -56,6 +56,7 @@ typedef enum {
wopRegisterDrvOutput = 42, wopRegisterDrvOutput = 42,
wopQueryRealisation = 43, wopQueryRealisation = 43,
wopAddMultipleToStore = 44, wopAddMultipleToStore = 44,
wopAddBuildLog = 45,
} WorkerOp; } WorkerOp;

View file

@ -4,7 +4,7 @@
using namespace nix; using namespace nix;
struct CmdCopy : CopyCommand struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand
{ {
CheckSigsFlag checkSigs = CheckSigs; CheckSigsFlag checkSigs = CheckSigs;
@ -45,8 +45,10 @@ struct CmdCopy : CopyCommand
Category category() override { return catSecondary; } Category category() override { return catSecondary; }
void run(ref<Store> srcStore, ref<Store> dstStore, BuiltPaths && paths) override void run(ref<Store> srcStore, BuiltPaths && paths) override
{ {
auto dstStore = getDstStore();
RealisedPath::Set stuffToCopy; RealisedPath::Set stuffToCopy;
for (auto & builtPath : paths) { for (auto & builtPath : paths) {

40
src/nix/store-copy-log.cc Normal file
View file

@ -0,0 +1,40 @@
#include "command.hh"
#include "shared.hh"
#include "store-api.hh"
#include "sync.hh"
#include "thread-pool.hh"
#include <atomic>
using namespace nix;
struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
{
std::string description() override
{
return "copy build logs between Nix stores";
}
std::string doc() override
{
return
#include "store-copy-log.md"
;
}
Category category() override { return catUtility; }
void run(ref<Store> srcStore) override
{
auto dstStore = getDstStore();
for (auto & path : toDerivations(srcStore, installables, true)) {
if (auto log = srcStore->getBuildLog(path))
dstStore->addBuildLog(path, *log);
else
throw Error("build log for '%s' is not available", srcStore->printStorePath(path));
}
}
};
static auto rCmdCopyLog = registerCommand2<CmdCopyLog>({"store", "copy-log"});

13
src/nix/store-copy-log.md Normal file
View file

@ -0,0 +1,13 @@
R""(
# Examples
TODO
# Description
`nix store copy-log` copies build logs between two Nix stores. The
source store is specified using `--from` and the destination using
`--to`. If one of these is omitted, it defaults to the local store.
)""