mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-27 00:06:16 +02:00
nix flake show: Speed up eval cache bigly
In the fully cached case for the 'nixpkgs' flake, it went from 101s to 4.6s. Populating the cache went from 132s to 17.4s (which could probably be improved further by combining INSERTs).
This commit is contained in:
Eelco Dolstra
parent
9ea4f93f88
commit
aa34c0ef51
1 changed files with 47 additions and 69 deletions
116
src/nix/flake.cc
116
src/nix/flake.cc
|
|
@ -669,21 +669,12 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
|
|||
}
|
||||
};
|
||||
|
||||
// FIXME: inefficient representation of attrs / fingerprints
|
||||
// FIXME: inefficient representation of attrs
|
||||
static const char * schema = R"sql(
|
||||
|
||||
create table if not exists Fingerprints (
|
||||
fingerprint blob primary key not null,
|
||||
timestamp integer not null
|
||||
);
|
||||
|
||||
create table if not exists Attributes (
|
||||
fingerprint blob not null,
|
||||
attrPath text not null,
|
||||
type integer,
|
||||
value text,
|
||||
primary key (fingerprint, attrPath),
|
||||
foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade
|
||||
attrPath text primary key,
|
||||
type integer not null,
|
||||
value text not null
|
||||
);
|
||||
)sql";
|
||||
|
||||
|
|
@ -697,72 +688,52 @@ struct AttrDb
|
|||
struct State
|
||||
{
|
||||
SQLite db;
|
||||
SQLiteStmt insertFingerprint;
|
||||
SQLiteStmt insertAttribute;
|
||||
SQLiteStmt queryAttribute;
|
||||
std::set<Fingerprint> fingerprints;
|
||||
};
|
||||
|
||||
std::unique_ptr<Sync<State>> _state;
|
||||
|
||||
AttrDb()
|
||||
AttrDb(const Fingerprint & fingerprint)
|
||||
: _state(std::make_unique<Sync<State>>())
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
Path dbPath = getCacheDir() + "/nix/eval-cache-v2.sqlite";
|
||||
createDirs(dirOf(dbPath));
|
||||
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
|
||||
createDirs(cacheDir);
|
||||
|
||||
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
state->insertFingerprint.create(state->db,
|
||||
"insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)");
|
||||
|
||||
state->insertAttribute.create(state->db,
|
||||
"insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)");
|
||||
"insert or replace into Attributes(attrPath, type, value) values (?, ?, ?)");
|
||||
|
||||
state->queryAttribute.create(state->db,
|
||||
"select type, value from Attributes where fingerprint = ? and attrPath = ?");
|
||||
}
|
||||
|
||||
void addFingerprint(State & state, const Fingerprint & fingerprint)
|
||||
{
|
||||
if (state.fingerprints.insert(fingerprint).second)
|
||||
// FIXME: update timestamp
|
||||
state.insertFingerprint.use()
|
||||
(fingerprint.hash, fingerprint.hashSize)
|
||||
(time(0)).exec();
|
||||
"select type, value from Attributes where attrPath = ?");
|
||||
}
|
||||
|
||||
void setAttr(
|
||||
const Fingerprint & fingerprint,
|
||||
const std::vector<Symbol> & attrPath,
|
||||
const std::vector<Symbol> & attrs)
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
addFingerprint(*state, fingerprint);
|
||||
|
||||
state->insertAttribute.use()
|
||||
(fingerprint.hash, fingerprint.hashSize)
|
||||
(concatStringsSep(".", attrPath))
|
||||
(AttrType::Attrs)
|
||||
(concatStringsSep("\n", attrs)).exec();
|
||||
}
|
||||
|
||||
void setAttr(
|
||||
const Fingerprint & fingerprint,
|
||||
const std::vector<Symbol> & attrPath,
|
||||
std::string_view s)
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
addFingerprint(*state, fingerprint);
|
||||
|
||||
state->insertAttribute.use()
|
||||
(fingerprint.hash, fingerprint.hashSize)
|
||||
(concatStringsSep(".", attrPath))
|
||||
(AttrType::String)
|
||||
(s).exec();
|
||||
|
|
@ -771,16 +742,12 @@ struct AttrDb
|
|||
typedef std::variant<std::vector<Symbol>, std::string> AttrValue;
|
||||
|
||||
std::optional<AttrValue> getAttr(
|
||||
const Fingerprint & fingerprint,
|
||||
const std::vector<Symbol> & attrPath,
|
||||
SymbolTable & symbols)
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
addFingerprint(*state, fingerprint);
|
||||
|
||||
auto queryAttribute(state->queryAttribute.use()
|
||||
(fingerprint.hash, fingerprint.hashSize)
|
||||
(concatStringsSep(".", attrPath)));
|
||||
if (!queryAttribute.next()) return {};
|
||||
|
||||
|
|
@ -804,15 +771,13 @@ struct AttrRoot : std::enable_shared_from_this<AttrRoot>
|
|||
{
|
||||
std::shared_ptr<AttrDb> db;
|
||||
EvalState & state;
|
||||
Fingerprint fingerprint;
|
||||
typedef std::function<Value *()> RootLoader;
|
||||
RootLoader rootLoader;
|
||||
RootValue value;
|
||||
|
||||
AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, const Fingerprint & fingerprint, RootLoader rootLoader)
|
||||
AttrRoot(std::shared_ptr<AttrDb> db, EvalState & state, RootLoader rootLoader)
|
||||
: db(db)
|
||||
, state(state)
|
||||
, fingerprint(fingerprint)
|
||||
, rootLoader(rootLoader)
|
||||
{
|
||||
}
|
||||
|
|
@ -838,12 +803,14 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
|||
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
|
||||
Parent parent;
|
||||
RootValue _value;
|
||||
std::optional<AttrDb::AttrValue> cachedValue;
|
||||
|
||||
AttrCursor(
|
||||
ref<AttrRoot> root,
|
||||
Parent parent,
|
||||
Value * value = nullptr)
|
||||
: root(root), parent(parent)
|
||||
Value * value = nullptr,
|
||||
std::optional<AttrDb::AttrValue> && cachedValue = {})
|
||||
: root(root), parent(parent), cachedValue(std::move(cachedValue))
|
||||
{
|
||||
if (value)
|
||||
_value = allocRootValue(value);
|
||||
|
|
@ -895,9 +862,11 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
|||
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
|
||||
{
|
||||
if (root->db) {
|
||||
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols);
|
||||
if (attr) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&*attr)) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
|
||||
|
||||
if (cachedValue) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&*cachedValue)) {
|
||||
for (auto & attr : *attrs)
|
||||
if (attr == name)
|
||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
|
||||
|
|
@ -905,10 +874,9 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
attr = root->db->getAttr(root->fingerprint, getAttrPath(name), root->state.symbols);
|
||||
auto attr = root->db->getAttr(getAttrPath(name), root->state.symbols);
|
||||
if (attr)
|
||||
// FIXME: store *attr
|
||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
|
||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
|
||||
}
|
||||
|
||||
//printError("GET ATTR %s", getAttrPathStr(name));
|
||||
|
|
@ -947,28 +915,36 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
|||
std::string getString()
|
||||
{
|
||||
if (root->db) {
|
||||
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols);
|
||||
if (auto s = std::get_if<std::string>(&*attr)) {
|
||||
//printError("GOT STRING %s", getAttrPathStr());
|
||||
return *s;
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
|
||||
if (cachedValue) {
|
||||
if (auto s = std::get_if<std::string>(&*cachedValue)) {
|
||||
//printError("GOT STRING %s", getAttrPathStr());
|
||||
return *s;
|
||||
} else
|
||||
throw Error("unexpected type mismatch in evaluation cache");
|
||||
}
|
||||
}
|
||||
|
||||
//printError("GET STRING %s", getAttrPathStr());
|
||||
auto s = root->state.forceString(getValue());
|
||||
if (root->db)
|
||||
root->db->setAttr(root->fingerprint, getAttrPath(), s);
|
||||
if (root->db) {
|
||||
root->db->setAttr(getAttrPath(), s);
|
||||
cachedValue = s;
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
std::vector<Symbol> getAttrs()
|
||||
{
|
||||
if (root->db) {
|
||||
auto attr = root->db->getAttr(root->fingerprint, getAttrPath(), root->state.symbols);
|
||||
if (attr) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&*attr)) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getAttrPath(), root->state.symbols);
|
||||
if (cachedValue) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&*cachedValue)) {
|
||||
//printError("GOT ATTRS %s", getAttrPathStr());
|
||||
return std::move(*attrs);
|
||||
return *attrs;
|
||||
} else
|
||||
throw Error("unexpected type mismatch in evaluation cache");
|
||||
}
|
||||
|
|
@ -982,8 +958,11 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
|||
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
|
||||
return (const string &) a < (const string &) b;
|
||||
});
|
||||
if (root->db)
|
||||
root->db->setAttr(root->fingerprint, getAttrPath(), attrs);
|
||||
if (root->db) {
|
||||
root->db->setAttr(getAttrPath(), attrs);
|
||||
cachedValue = attrs;
|
||||
}
|
||||
|
||||
return attrs;
|
||||
}
|
||||
|
||||
|
|
@ -1132,10 +1111,9 @@ struct CmdFlakeShow : FlakeCommand
|
|||
}
|
||||
};
|
||||
|
||||
auto db = std::make_shared<AttrDb>();
|
||||
auto db = std::make_shared<AttrDb>(flake.getFingerprint());
|
||||
|
||||
auto root = std::make_shared<AttrRoot>(db, *state,
|
||||
flake.getFingerprint(),
|
||||
[&]()
|
||||
{
|
||||
auto vFlake = state->allocValue();
|
||||
|
|
|
|||
Loading…
Reference in a new issue