Add "nix verify-paths" command

Unlike "nix-store --verify-path", this command verifies signatures in
addition to store path contents, is multi-threaded (especially useful
when verifying binary caches), and has a progress indicator.

Example use:

$ nix verify-paths --store https://cache.nixos.org -r $(type -p thunderbird)
...
[17/132 checked] checking ‘/nix/store/rawakphadqrqxr6zri2rmnxh03gqkrl3-autogen-5.18.6’
This commit is contained in:
Eelco Dolstra 2016-03-29 14:29:50 +02:00
parent 0ebe69dc67
commit 784ee35c80
11 changed files with 432 additions and 2 deletions

View file

@ -1,5 +1,6 @@
#include "crypto.hh" #include "crypto.hh"
#include "util.hh" #include "util.hh"
#include "globals.hh"
#if HAVE_SODIUM #if HAVE_SODIUM
#include <sodium.h> #include <sodium.h>
@ -98,4 +99,15 @@ bool verifyDetached(const std::string & data, const std::string & sig,
#endif #endif
} }
PublicKeys getDefaultPublicKeys()
{
PublicKeys publicKeys;
for (auto s : settings.get("binary-cache-public-keys", Strings())) {
PublicKey key(s);
publicKeys.emplace(key.name, key);
// FIXME: filter duplicates
}
return publicKeys;
}
} }

View file

@ -49,4 +49,6 @@ typedef std::map<std::string, PublicKey> PublicKeys;
bool verifyDetached(const std::string & data, const std::string & sig, bool verifyDetached(const std::string & data, const std::string & sig,
const PublicKeys & publicKeys); const PublicKeys & publicKeys);
PublicKeys getDefaultPublicKeys();
} }

View file

@ -0,0 +1,81 @@
#include "thread-pool.hh"
namespace nix {
ThreadPool::ThreadPool(size_t _nrThreads)
: nrThreads(_nrThreads)
{
if (!nrThreads) {
nrThreads = std::thread::hardware_concurrency();
if (!nrThreads) nrThreads = 1;
}
}
void ThreadPool::enqueue(const work_t & t)
{
auto state_(state.lock());
state_->left.push(t);
wakeup.notify_one();
}
void ThreadPool::process()
{
printMsg(lvlDebug, format("starting pool of %d threads") % nrThreads);
std::vector<std::thread> workers;
for (size_t n = 0; n < nrThreads; n++)
workers.push_back(std::thread([&]() {
bool first = true;
while (true) {
work_t work;
{
auto state_(state.lock());
if (state_->exception) return;
if (!first) {
assert(state_->pending);
state_->pending--;
}
first = false;
while (state_->left.empty()) {
if (!state_->pending) {
wakeup.notify_all();
return;
}
if (state_->exception) return;
state_.wait(wakeup);
}
work = state_->left.front();
state_->left.pop();
state_->pending++;
}
try {
work();
} catch (std::exception & e) {
auto state_(state.lock());
if (state_->exception)
printMsg(lvlError, format("error: %s") % e.what());
else {
state_->exception = std::current_exception();
wakeup.notify_all();
}
}
}
}));
for (auto & thr : workers)
thr.join();
{
auto state_(state.lock());
if (state_->exception)
std::rethrow_exception(state_->exception);
}
}
}

View file

@ -0,0 +1,52 @@
#pragma once
#include "sync.hh"
#include "util.hh"
#include <queue>
#include <functional>
#include <thread>
namespace nix {
/* A simple thread pool that executes a queue of work items
(lambdas). */
class ThreadPool
{
public:
ThreadPool(size_t nrThreads = 0);
// FIXME: use std::packaged_task?
typedef std::function<void()> work_t;
/* Enqueue a function to be executed by the thread pool. */
void enqueue(const work_t & t);
/* Execute work items until the queue is empty. Note that work
items are allowed to add new items to the queue; this is
handled correctly. Queue processing stops prematurely if any
work item throws an exception. This exception is propagated to
the calling thread. If multiple work items throw an exception
concurrently, only one item is propagated; the others are
printed on stderr and otherwise ignored. */
void process();
private:
size_t nrThreads;
struct State
{
std::queue<work_t> left;
size_t pending = 0;
std::exception_ptr exception;
};
Sync<State> state;
std::condition_variable wakeup;
};
}

View file

@ -548,7 +548,7 @@ void writeToStderr(const string & s)
} }
void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; std::function<void(const unsigned char * buf, size_t count)> _writeToStderr;
void readFull(int fd, unsigned char * buf, size_t count) void readFull(int fd, unsigned char * buf, size_t count)

View file

@ -167,7 +167,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs);
void writeToStderr(const string & s); void writeToStderr(const string & s);
extern void (*_writeToStderr) (const unsigned char * buf, size_t count); extern std::function<void(const unsigned char * buf, size_t count)> _writeToStderr;
/* Wrappers arount read()/write() that read/write exactly the /* Wrappers arount read()/write() that read/write exactly the

View file

@ -69,4 +69,25 @@ void StoreCommand::run()
run(openStoreAt(storeUri)); run(openStoreAt(storeUri));
} }
StorePathsCommand::StorePathsCommand()
{
expectArgs("paths", &storePaths);
mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive);
}
void StorePathsCommand::run(ref<Store> store)
{
for (auto & storePath : storePaths)
storePath = followLinksToStorePath(storePath);
if (recursive) {
PathSet closure;
for (auto & storePath : storePaths)
store->computeFSClosure(storePath, closure, false, false);
storePaths = store->topoSortPaths(closure);
}
run(store, storePaths);
}
} }

View file

@ -24,6 +24,23 @@ struct StoreCommand : virtual Command
virtual void run(ref<Store>) = 0; virtual void run(ref<Store>) = 0;
}; };
/* A command that operates on zero or more store paths. */
struct StorePathsCommand : public StoreCommand
{
private:
Paths storePaths;
bool recursive = false;
public:
StorePathsCommand();
virtual void run(ref<Store> store, Paths storePaths) = 0;
void run(ref<Store> store) override;
};
typedef std::map<std::string, ref<Command>> Commands; typedef std::map<std::string, ref<Command>> Commands;
/* An argument parser that supports multiple subcommands, /* An argument parser that supports multiple subcommands,

72
src/nix/progress-bar.cc Normal file
View file

@ -0,0 +1,72 @@
#include "progress-bar.hh"
#include <iostream>
namespace nix {
ProgressBar::ProgressBar()
{
_writeToStderr = [&](const unsigned char * buf, size_t count) {
auto state_(state.lock());
assert(!state_->done);
std::cerr << "\r\e[K" << std::string((const char *) buf, count);
render(*state_);
};
}
ProgressBar::~ProgressBar()
{
done();
}
void ProgressBar::updateStatus(const std::string & s)
{
auto state_(state.lock());
assert(!state_->done);
state_->status = s;
render(*state_);
}
void ProgressBar::done()
{
auto state_(state.lock());
assert(state_->activities.empty());
state_->done = true;
std::cerr << "\r\e[K";
std::cerr.flush();
_writeToStderr = decltype(_writeToStderr)();
}
void ProgressBar::render(State & state_)
{
std::cerr << '\r' << state_.status;
if (!state_.activities.empty()) {
if (!state_.status.empty()) std::cerr << ' ';
std::cerr << *state_.activities.rbegin();
}
std::cerr << "\e[K";
std::cerr.flush();
}
ProgressBar::Activity ProgressBar::startActivity(const FormatOrString & fs)
{
return Activity(*this, fs);
}
ProgressBar::Activity::Activity(ProgressBar & pb, const FormatOrString & fs)
: pb(pb)
{
auto state_(pb.state.lock());
state_->activities.push_back(fs.s);
it = state_->activities.end(); --it;
pb.render(*state_);
}
ProgressBar::Activity::~Activity()
{
auto state_(pb.state.lock());
state_->activities.erase(it);
}
}

49
src/nix/progress-bar.hh Normal file
View file

@ -0,0 +1,49 @@
#pragma once
#include "sync.hh"
#include "util.hh"
namespace nix {
class ProgressBar
{
private:
struct State
{
std::string status;
bool done = false;
std::list<std::string> activities;
};
Sync<State> state;
public:
ProgressBar();
~ProgressBar();
void updateStatus(const std::string & s);
void done();
class Activity
{
friend class ProgressBar;
private:
ProgressBar & pb;
std::list<std::string>::iterator it;
Activity(ProgressBar & pb, const FormatOrString & fs);
public:
~Activity();
};
Activity startActivity(const FormatOrString & fs);
private:
void render(State & state_);
};
}

124
src/nix/verify.cc Normal file
View file

@ -0,0 +1,124 @@
#include "affinity.hh" // FIXME
#include "command.hh"
#include "progress-bar.hh"
#include "shared.hh"
#include "store-api.hh"
#include "sync.hh"
#include "thread-pool.hh"
#include <atomic>
using namespace nix;
struct CmdVerifyPaths : StorePathsCommand
{
bool noContents = false;
bool noSigs = false;
CmdVerifyPaths()
{
mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
mkFlag(0, "no-sigs", "do not verify whether each store path has a valid signature", &noSigs);
}
std::string name() override
{
return "verify-paths";
}
std::string description() override
{
return "verify the integrity of store paths";
}
void run(ref<Store> store, Paths storePaths) override
{
restoreAffinity(); // FIXME
auto publicKeys = getDefaultPublicKeys();
std::atomic<size_t> untrusted{0};
std::atomic<size_t> corrupted{0};
std::atomic<size_t> done{0};
std::atomic<size_t> failed{0};
ProgressBar progressBar;
auto showProgress = [&](bool final) {
std::string s;
if (final)
s = (format("checked %d paths") % storePaths.size()).str();
else
s = (format("[%d/%d checked") % done % storePaths.size()).str();
if (corrupted > 0)
s += (format(", %d corrupted") % corrupted).str();
if (untrusted > 0)
s += (format(", %d untrusted") % untrusted).str();
if (failed > 0)
s += (format(", %d failed") % failed).str();
if (!final) s += "]";
return s;
};
progressBar.updateStatus(showProgress(false));
ThreadPool pool;
auto doPath = [&](const Path & storePath) {
try {
progressBar.startActivity(format("checking %s") % storePath);
auto info = store->queryPathInfo(storePath);
if (!noContents) {
HashSink sink(info.narHash.type);
store->narFromPath(storePath, sink);
auto hash = sink.finish();
if (hash.first != info.narHash) {
corrupted = 1;
printMsg(lvlError,
format("path %s was modified! expected hash %s, got %s")
% storePath % printHash(info.narHash) % printHash(hash.first));
}
}
if (!noSigs) {
if (!info.ultimate && !info.checkSignatures(publicKeys)) {
untrusted++;
printMsg(lvlError, format("path %s is untrusted") % storePath);
}
}
done++;
progressBar.updateStatus(showProgress(false));
} catch (Error & e) {
printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
failed++;
}
};
for (auto & storePath : storePaths)
pool.enqueue(std::bind(doPath, storePath));
pool.process();
progressBar.done();
printMsg(lvlInfo, showProgress(true));
throw Exit(
(corrupted ? 1 : 0) |
(untrusted ? 2 : 0) |
(failed ? 4 : 0));
}
};
static RegisterCommand r1(make_ref<CmdVerifyPaths>());