mirror of
https://github.com/privatevoid-net/nix-super.git
synced 2024-11-10 16:26:18 +02:00
Store more stuff in the evaluation cache
In particular, we store whether an attribute failed to evaluate (threw an exception) or was an unsupported type. This is to ensure that a repeated 'nix flake show' never has to evaluate anything, so it can execute without fetching the flake. With this, 'nix flake show nixpkgs/nixos-20.03 --legacy' executes in 0.6s (was 3.4s).
This commit is contained in:
parent
3738bcb05e
commit
0725ab2fd7
1 changed files with 152 additions and 37 deletions
189
src/nix/flake.cc
189
src/nix/flake.cc
|
@ -682,9 +682,11 @@ create table if not exists Attributes (
|
||||||
|
|
||||||
enum AttrType {
|
enum AttrType {
|
||||||
Placeholder = 0,
|
Placeholder = 0,
|
||||||
// FIXME: distinguish between full / partial attrsets
|
FullAttrs = 1,
|
||||||
Attrs = 1,
|
|
||||||
String = 2,
|
String = 2,
|
||||||
|
Missing = 3,
|
||||||
|
Misc = 4,
|
||||||
|
Failed = 5,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct AttrDb
|
struct AttrDb
|
||||||
|
@ -693,16 +695,18 @@ struct AttrDb
|
||||||
{
|
{
|
||||||
SQLite db;
|
SQLite db;
|
||||||
SQLiteStmt insertAttribute;
|
SQLiteStmt insertAttribute;
|
||||||
SQLiteStmt insertPlaceholder;
|
|
||||||
SQLiteStmt queryAttribute;
|
SQLiteStmt queryAttribute;
|
||||||
SQLiteStmt queryAttributes;
|
SQLiteStmt queryAttributes;
|
||||||
std::unique_ptr<SQLiteTxn> txn;
|
std::unique_ptr<SQLiteTxn> txn;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct placeholder_t {};
|
struct placeholder_t {};
|
||||||
|
struct missing_t {};
|
||||||
|
struct misc_t {};
|
||||||
|
struct failed_t {};
|
||||||
typedef uint64_t AttrId;
|
typedef uint64_t AttrId;
|
||||||
typedef std::pair<AttrId, Symbol> AttrKey;
|
typedef std::pair<AttrId, Symbol> AttrKey;
|
||||||
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t> AttrValue;
|
typedef std::variant<std::vector<Symbol>, std::string, placeholder_t, missing_t, misc_t, failed_t> AttrValue;
|
||||||
|
|
||||||
std::unique_ptr<Sync<State>> _state;
|
std::unique_ptr<Sync<State>> _state;
|
||||||
|
|
||||||
|
@ -723,9 +727,6 @@ struct AttrDb
|
||||||
state->insertAttribute.create(state->db,
|
state->insertAttribute.create(state->db,
|
||||||
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
|
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
|
||||||
|
|
||||||
state->insertPlaceholder.create(state->db,
|
|
||||||
fmt("insert or ignore into Attributes(parent, name, type) values (?, ?, %d)", Placeholder));
|
|
||||||
|
|
||||||
state->queryAttribute.create(state->db,
|
state->queryAttribute.create(state->db,
|
||||||
"select rowid, type, value from Attributes where parent = ? and name = ?");
|
"select rowid, type, value from Attributes where parent = ? and name = ?");
|
||||||
|
|
||||||
|
@ -746,7 +747,7 @@ struct AttrDb
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AttrId setAttr(
|
AttrId setAttrs(
|
||||||
AttrKey key,
|
AttrKey key,
|
||||||
const std::vector<Symbol> & attrs)
|
const std::vector<Symbol> & attrs)
|
||||||
{
|
{
|
||||||
|
@ -755,21 +756,23 @@ struct AttrDb
|
||||||
state->insertAttribute.use()
|
state->insertAttribute.use()
|
||||||
(key.first)
|
(key.first)
|
||||||
(key.second)
|
(key.second)
|
||||||
(AttrType::Attrs)
|
(AttrType::FullAttrs)
|
||||||
(0, false).exec();
|
(0, false).exec();
|
||||||
|
|
||||||
AttrId rowId = state->db.getLastInsertedRowId();
|
AttrId rowId = state->db.getLastInsertedRowId();
|
||||||
assert(rowId);
|
assert(rowId);
|
||||||
|
|
||||||
for (auto & attr : attrs)
|
for (auto & attr : attrs)
|
||||||
state->insertPlaceholder.use()
|
state->insertAttribute.use()
|
||||||
(rowId)
|
(rowId)
|
||||||
(attr).exec();
|
(attr)
|
||||||
|
(AttrType::Placeholder)
|
||||||
|
(0, false).exec();
|
||||||
|
|
||||||
return rowId;
|
return rowId;
|
||||||
}
|
}
|
||||||
|
|
||||||
AttrId setAttr(
|
AttrId setString(
|
||||||
AttrKey key,
|
AttrKey key,
|
||||||
std::string_view s)
|
std::string_view s)
|
||||||
{
|
{
|
||||||
|
@ -784,6 +787,58 @@ struct AttrDb
|
||||||
return state->db.getLastInsertedRowId();
|
return state->db.getLastInsertedRowId();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
AttrId setPlaceholder(AttrKey key)
|
||||||
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
|
||||||
|
state->insertAttribute.use()
|
||||||
|
(key.first)
|
||||||
|
(key.second)
|
||||||
|
(AttrType::Placeholder)
|
||||||
|
(0, false).exec();
|
||||||
|
|
||||||
|
return state->db.getLastInsertedRowId();
|
||||||
|
}
|
||||||
|
|
||||||
|
AttrId setMissing(AttrKey key)
|
||||||
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
|
||||||
|
state->insertAttribute.use()
|
||||||
|
(key.first)
|
||||||
|
(key.second)
|
||||||
|
(AttrType::Missing)
|
||||||
|
(0, false).exec();
|
||||||
|
|
||||||
|
return state->db.getLastInsertedRowId();
|
||||||
|
}
|
||||||
|
|
||||||
|
AttrId setMisc(AttrKey key)
|
||||||
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
|
||||||
|
state->insertAttribute.use()
|
||||||
|
(key.first)
|
||||||
|
(key.second)
|
||||||
|
(AttrType::Misc)
|
||||||
|
(0, false).exec();
|
||||||
|
|
||||||
|
return state->db.getLastInsertedRowId();
|
||||||
|
}
|
||||||
|
|
||||||
|
AttrId setFailed(AttrKey key)
|
||||||
|
{
|
||||||
|
auto state(_state->lock());
|
||||||
|
|
||||||
|
state->insertAttribute.use()
|
||||||
|
(key.first)
|
||||||
|
(key.second)
|
||||||
|
(AttrType::Failed)
|
||||||
|
(0, false).exec();
|
||||||
|
|
||||||
|
return state->db.getLastInsertedRowId();
|
||||||
|
}
|
||||||
|
|
||||||
std::optional<std::pair<AttrId, AttrValue>> getAttr(
|
std::optional<std::pair<AttrId, AttrValue>> getAttr(
|
||||||
AttrKey key,
|
AttrKey key,
|
||||||
SymbolTable & symbols)
|
SymbolTable & symbols)
|
||||||
|
@ -798,7 +853,7 @@ struct AttrDb
|
||||||
|
|
||||||
if (type == AttrType::Placeholder)
|
if (type == AttrType::Placeholder)
|
||||||
return {{rowId, placeholder_t()}};
|
return {{rowId, placeholder_t()}};
|
||||||
else if (type == AttrType::Attrs) {
|
else if (type == AttrType::FullAttrs) {
|
||||||
// FIXME: expensive, should separate this out.
|
// FIXME: expensive, should separate this out.
|
||||||
std::vector<Symbol> attrs;
|
std::vector<Symbol> attrs;
|
||||||
auto queryAttributes(state->queryAttributes.use()(rowId));
|
auto queryAttributes(state->queryAttributes.use()(rowId));
|
||||||
|
@ -807,6 +862,12 @@ struct AttrDb
|
||||||
return {{rowId, attrs}};
|
return {{rowId, attrs}};
|
||||||
} else if (type == AttrType::String) {
|
} else if (type == AttrType::String) {
|
||||||
return {{rowId, queryAttribute.getStr(2)}};
|
return {{rowId, queryAttribute.getStr(2)}};
|
||||||
|
} else if (type == AttrType::Missing) {
|
||||||
|
return {{rowId, missing_t()}};
|
||||||
|
} else if (type == AttrType::Misc) {
|
||||||
|
return {{rowId, misc_t()}};
|
||||||
|
} else if (type == AttrType::Failed) {
|
||||||
|
return {{rowId, failed_t()}};
|
||||||
} else
|
} else
|
||||||
throw Error("unexpected type in evaluation cache");
|
throw Error("unexpected type in evaluation cache");
|
||||||
}
|
}
|
||||||
|
@ -832,7 +893,7 @@ struct AttrRoot : std::enable_shared_from_this<AttrRoot>
|
||||||
Value * getRootValue()
|
Value * getRootValue()
|
||||||
{
|
{
|
||||||
if (!value) {
|
if (!value) {
|
||||||
//printError("GET ROOT");
|
debug("getting root value");
|
||||||
value = allocRootValue(rootLoader());
|
value = allocRootValue(rootLoader());
|
||||||
}
|
}
|
||||||
return *value;
|
return *value;
|
||||||
|
@ -918,6 +979,33 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
||||||
return concatStringsSep(".", getAttrPath(name));
|
return concatStringsSep(".", getAttrPath(name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Value & forceValue()
|
||||||
|
{
|
||||||
|
debug("evaluating uncached attribute %s", getAttrPathStr());
|
||||||
|
|
||||||
|
auto & v = getValue();
|
||||||
|
|
||||||
|
try {
|
||||||
|
root->state.forceValue(v);
|
||||||
|
} catch (EvalError &) {
|
||||||
|
debug("setting '%s' to failed", getAttrPathStr());
|
||||||
|
if (root->db)
|
||||||
|
cachedValue = {root->db->setFailed(getAttrKey()), AttrDb::failed_t()};
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (root->db && (!cachedValue || std::get_if<AttrDb::placeholder_t>(&cachedValue->second))) {
|
||||||
|
if (v.type == tString)
|
||||||
|
cachedValue = {root->db->setString(getAttrKey(), v.string.s), v.string.s};
|
||||||
|
else if (v.type == tAttrs)
|
||||||
|
; // FIXME: do something?
|
||||||
|
else
|
||||||
|
cachedValue = {root->db->setMisc(getAttrKey()), AttrDb::misc_t()};
|
||||||
|
}
|
||||||
|
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
|
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name)
|
||||||
{
|
{
|
||||||
if (root->db) {
|
if (root->db) {
|
||||||
|
@ -932,29 +1020,47 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
||||||
return nullptr;
|
return nullptr;
|
||||||
} else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
} else if (std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
||||||
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
|
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
|
||||||
if (attr)
|
if (attr) {
|
||||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
|
if (std::get_if<AttrDb::missing_t>(&attr->second))
|
||||||
|
return nullptr;
|
||||||
|
else if (std::get_if<AttrDb::failed_t>(&attr->second))
|
||||||
|
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
|
||||||
|
else
|
||||||
|
return std::make_shared<AttrCursor>(root,
|
||||||
|
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
|
||||||
|
}
|
||||||
// Incomplete attrset, so need to fall thru and
|
// Incomplete attrset, so need to fall thru and
|
||||||
// evaluate to see whether 'name' exists
|
// evaluate to see whether 'name' exists
|
||||||
} else
|
} else
|
||||||
// FIXME: throw error?
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//printError("GET ATTR %s", getAttrPathStr(name));
|
auto & v = forceValue();
|
||||||
|
|
||||||
root->state.forceValue(getValue());
|
if (v.type != tAttrs)
|
||||||
|
|
||||||
if (getValue().type != tAttrs)
|
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||||
|
|
||||||
auto attr = getValue().attrs->get(name);
|
auto attr = v.attrs->get(name);
|
||||||
|
|
||||||
if (!attr)
|
if (!attr) {
|
||||||
|
if (root->db) {
|
||||||
|
assert(cachedValue);
|
||||||
|
root->db->setMissing({cachedValue->first, name});
|
||||||
|
}
|
||||||
return nullptr;
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name), attr->value);
|
std::optional<std::pair<AttrDb::AttrId, AttrDb::AttrValue>> cachedValue2;
|
||||||
|
if (root->db) {
|
||||||
|
assert(cachedValue);
|
||||||
|
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), AttrDb::placeholder_t()};
|
||||||
|
}
|
||||||
|
|
||||||
|
return std::make_shared<AttrCursor>(
|
||||||
|
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
|
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name)
|
||||||
|
@ -982,19 +1088,19 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
||||||
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
|
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
|
||||||
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
||||||
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
|
if (auto s = std::get_if<std::string>(&cachedValue->second)) {
|
||||||
//printError("GOT STRING %s", getAttrPathStr());
|
debug("using cached string attribute '%s'", getAttrPathStr());
|
||||||
return *s;
|
return *s;
|
||||||
} else
|
} else
|
||||||
throw Error("unexpected type mismatch in evaluation cache (string expected)");
|
throw TypeError("'%s' is not a string", getAttrPathStr());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//printError("GET STRING %s", getAttrPathStr());
|
auto & v = forceValue();
|
||||||
auto s = root->state.forceString(getValue());
|
|
||||||
if (root->db)
|
|
||||||
cachedValue = {root->db->setAttr(getAttrKey(), s), s};
|
|
||||||
|
|
||||||
return s;
|
if (v.type != tString)
|
||||||
|
throw TypeError("'%s' is not a string", getAttrPathStr());
|
||||||
|
|
||||||
|
return v.string.s;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<Symbol> getAttrs()
|
std::vector<Symbol> getAttrs()
|
||||||
|
@ -1004,23 +1110,27 @@ struct AttrCursor : std::enable_shared_from_this<AttrCursor>
|
||||||
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
|
cachedValue = root->db->getAttr(getAttrKey(), root->state.symbols);
|
||||||
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
if (cachedValue && !std::get_if<AttrDb::placeholder_t>(&cachedValue->second)) {
|
||||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
|
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
|
||||||
//printError("GOT ATTRS %s", getAttrPathStr());
|
debug("using cached attrset attribute '%s'", getAttrPathStr());
|
||||||
return *attrs;
|
return *attrs;
|
||||||
} else
|
} else
|
||||||
throw Error("unexpected type mismatch in evaluation cache (attrs expected)");
|
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//printError("GET ATTRS %s", getAttrPathStr());
|
auto & v = forceValue();
|
||||||
|
|
||||||
|
if (v.type != tAttrs)
|
||||||
|
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||||
|
|
||||||
std::vector<Symbol> attrs;
|
std::vector<Symbol> attrs;
|
||||||
root->state.forceAttrs(getValue());
|
|
||||||
for (auto & attr : *getValue().attrs)
|
for (auto & attr : *getValue().attrs)
|
||||||
attrs.push_back(attr.name);
|
attrs.push_back(attr.name);
|
||||||
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
|
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
|
||||||
return (const string &) a < (const string &) b;
|
return (const string &) a < (const string &) b;
|
||||||
});
|
});
|
||||||
|
|
||||||
if (root->db)
|
if (root->db)
|
||||||
cachedValue = {root->db->setAttr(getAttrKey(), attrs), attrs};
|
cachedValue = {root->db->setAttrs(getAttrKey(), attrs), attrs};
|
||||||
|
|
||||||
return attrs;
|
return attrs;
|
||||||
}
|
}
|
||||||
|
@ -1172,7 +1282,7 @@ struct CmdFlakeShow : FlakeCommand
|
||||||
}
|
}
|
||||||
} catch (EvalError & e) {
|
} catch (EvalError & e) {
|
||||||
if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
|
if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
|
||||||
logger->stdout("%s: " ANSI_RED "%s" ANSI_NORMAL, headerPrefix, e.what());
|
throw;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1181,6 +1291,11 @@ struct CmdFlakeShow : FlakeCommand
|
||||||
auto root = std::make_shared<AttrRoot>(db, *state,
|
auto root = std::make_shared<AttrRoot>(db, *state,
|
||||||
[&]()
|
[&]()
|
||||||
{
|
{
|
||||||
|
/* For testing whether the evaluation cache is
|
||||||
|
complete. */
|
||||||
|
if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
|
||||||
|
throw Error("not everything is cached, but evaluation is not allowed");
|
||||||
|
|
||||||
auto vFlake = state->allocValue();
|
auto vFlake = state->allocValue();
|
||||||
flake::callFlake(*state, flake, *vFlake);
|
flake::callFlake(*state, flake, *vFlake);
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue