#include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" namespace nix::eval_cache { static const char * schema = R"sql( create table if not exists Attributes ( parent integer not null, name text, type integer not null, value text, context text, primary key (parent, name) ); )sql"; struct AttrDb { std::atomic_bool failed{false}; struct State { SQLite db; SQLiteStmt insertAttribute; SQLiteStmt insertAttributeWithContext; SQLiteStmt queryAttribute; SQLiteStmt queryAttributes; std::unique_ptr txn; }; std::unique_ptr> _state; AttrDb(const Hash & fingerprint) : _state(std::make_unique>()) { auto state(_state->lock()); Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); state->db.exec(schema); state->insertAttribute.create(state->db, "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); state->insertAttributeWithContext.create(state->db, "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); state->queryAttribute.create(state->db, "select rowid, type, value, context from Attributes where parent = ? and name = ?"); state->queryAttributes.create(state->db, "select name from Attributes where parent = ?"); state->txn = std::make_unique(state->db); } ~AttrDb() { try { auto state(_state->lock()); if (!failed) state->txn->commit(); state->txn.reset(); } catch (...) { ignoreException(); } } template AttrId doSQLite(F && fun) { if (failed) return 0; try { return fun(); } catch (SQLiteError &) { ignoreException(); failed = true; return 0; } } AttrId setAttrs( AttrKey key, const std::vector & attrs) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::FullAttrs) (0, false).exec(); AttrId rowId = state->db.getLastInsertedRowId(); assert(rowId); for (auto & attr : attrs) state->insertAttribute.use() (rowId) (attr) (AttrType::Placeholder) (0, false).exec(); return rowId; }); } AttrId setString( AttrKey key, std::string_view s, const char * * context = nullptr) { return doSQLite([&]() { auto state(_state->lock()); if (context) { std::string ctx; for (const char * * p = context; *p; ++p) { if (p != context) ctx.push_back(' '); ctx.append(*p); } state->insertAttributeWithContext.use() (key.first) (key.second) (AttrType::String) (s) (ctx).exec(); } else { state->insertAttribute.use() (key.first) (key.second) (AttrType::String) (s).exec(); } return state->db.getLastInsertedRowId(); }); } AttrId setBool( AttrKey key, bool b) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::Bool) (b ? 1 : 0).exec(); return state->db.getLastInsertedRowId(); }); } AttrId setPlaceholder(AttrKey key) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::Placeholder) (0, false).exec(); return state->db.getLastInsertedRowId(); }); } AttrId setMissing(AttrKey key) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::Missing) (0, false).exec(); return state->db.getLastInsertedRowId(); }); } AttrId setMisc(AttrKey key) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::Misc) (0, false).exec(); return state->db.getLastInsertedRowId(); }); } AttrId setFailed(AttrKey key) { return doSQLite([&]() { auto state(_state->lock()); state->insertAttribute.use() (key.first) (key.second) (AttrType::Failed) (0, false).exec(); return state->db.getLastInsertedRowId(); }); } std::optional> getAttr( AttrKey key, SymbolTable & symbols) { auto state(_state->lock()); auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); if (!queryAttribute.next()) return {}; auto rowId = (AttrType) queryAttribute.getInt(0); auto type = (AttrType) queryAttribute.getInt(1); switch (type) { case AttrType::Placeholder: return {{rowId, placeholder_t()}}; case AttrType::FullAttrs: { // FIXME: expensive, should separate this out. std::vector attrs; auto queryAttributes(state->queryAttributes.use()(rowId)); while (queryAttributes.next()) attrs.push_back(symbols.create(queryAttributes.getStr(0))); return {{rowId, attrs}}; } case AttrType::String: { std::vector> context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) context.push_back(decodeContext(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Missing: return {{rowId, missing_t()}}; case AttrType::Misc: return {{rowId, misc_t()}}; case AttrType::Failed: return {{rowId, failed_t()}}; default: throw Error("unexpected type in evaluation cache"); } } }; static std::shared_ptr makeAttrDb(const Hash & fingerprint) { try { return std::make_shared(fingerprint); } catch (SQLiteError &) { ignoreException(); return nullptr; } } EvalCache::EvalCache( std::optional> useCache, EvalState & state, RootLoader rootLoader) : db(useCache ? makeAttrDb(*useCache) : nullptr) , state(state) , rootLoader(rootLoader) { } Value * EvalCache::getRootValue() { if (!value) { debug("getting root value"); value = allocRootValue(rootLoader()); } return *value; } std::shared_ptr EvalCache::getRoot() { return std::make_shared(ref(shared_from_this()), std::nullopt); } AttrCursor::AttrCursor( ref root, Parent parent, Value * value, std::optional> && cachedValue) : root(root), parent(parent), cachedValue(std::move(cachedValue)) { if (value) _value = allocRootValue(value); } AttrKey AttrCursor::getKey() { if (!parent) return {0, root->state.sEpsilon}; if (!parent->first->cachedValue) { parent->first->cachedValue = root->db->getAttr( parent->first->getKey(), root->state.symbols); assert(parent->first->cachedValue); } return {parent->first->cachedValue->first, parent->second}; } Value & AttrCursor::getValue() { if (!_value) { if (parent) { auto & vParent = parent->first->getValue(); root->state.forceAttrs(vParent); auto attr = vParent.attrs->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); _value = allocRootValue(attr->value); } else _value = allocRootValue(root->getRootValue()); } return **_value; } std::vector AttrCursor::getAttrPath() const { if (parent) { auto attrPath = parent->first->getAttrPath(); attrPath.push_back(parent->second); return attrPath; } else return {}; } std::vector AttrCursor::getAttrPath(Symbol name) const { auto attrPath = getAttrPath(); attrPath.push_back(name); return attrPath; } std::string AttrCursor::getAttrPathStr() const { return concatStringsSep(".", getAttrPath()); } std::string AttrCursor::getAttrPathStr(Symbol name) const { return concatStringsSep(".", getAttrPath(name)); } Value & AttrCursor::forceValue() { debug("evaluating uncached attribute %s", getAttrPathStr()); auto & v = getValue(); try { root->state.forceValue(v); } catch (EvalError &) { debug("setting '%s' to failed", getAttrPathStr()); if (root->db) cachedValue = {root->db->setFailed(getKey()), failed_t()}; throw; } if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type == tString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; else if (v.type == tPath) cachedValue = {root->db->setString(getKey(), v.path), v.path}; else if (v.type == tBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type == tAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; } return v; } std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue) { if (auto attrs = std::get_if>(&cachedValue->second)) { for (auto & attr : *attrs) if (attr == name) return std::make_shared(root, std::make_pair(shared_from_this(), name)); return nullptr; } else if (std::get_if(&cachedValue->second)) { auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); if (attr) { if (std::get_if(&attr->second)) return nullptr; else if (std::get_if(&attr->second)) { if (forceErrors) debug("reevaluating failed cached attribute '%s'"); else throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); } else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } // Incomplete attrset, so need to fall thru and // evaluate to see whether 'name' exists } else return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); } } auto & v = forceValue(); if (v.type != tAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); auto attr = v.attrs->get(name); if (!attr) { if (root->db) { if (!cachedValue) cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; root->db->setMissing({cachedValue->first, name}); } return nullptr; } std::optional> cachedValue2; if (root->db) { if (!cachedValue) cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; } return std::make_shared( root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); } std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) { return maybeGetAttr(root->state.symbols.create(name)); } std::shared_ptr AttrCursor::getAttr(Symbol name, bool forceErrors) { auto p = maybeGetAttr(name, forceErrors); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return p; } std::shared_ptr AttrCursor::getAttr(std::string_view name) { return getAttr(root->state.symbols.create(name)); } std::shared_ptr AttrCursor::findAlongAttrPath(const std::vector & attrPath) { auto res = shared_from_this(); for (auto & attr : attrPath) { res = res->maybeGetAttr(attr); if (!res) return {}; } return res; } std::string AttrCursor::getString() { if (root->db) { if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else throw TypeError("'%s' is not a string", getAttrPathStr()); } } auto & v = forceValue(); if (v.type != tString && v.type != tPath) throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); return v.type == tString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() { if (root->db) { if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto s = std::get_if(&cachedValue->second)) { debug("using cached string attribute '%s'", getAttrPathStr()); return *s; } else throw TypeError("'%s' is not a string", getAttrPathStr()); } } auto & v = forceValue(); if (v.type == tString) return {v.string.s, v.getContext()}; else if (v.type == tPath) return {v.path, {}}; else throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); } bool AttrCursor::getBool() { if (root->db) { if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto b = std::get_if(&cachedValue->second)) { debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else throw TypeError("'%s' is not a Boolean", getAttrPathStr()); } } auto & v = forceValue(); if (v.type != tBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; } std::vector AttrCursor::getAttrs() { if (root->db) { if (!cachedValue) cachedValue = root->db->getAttr(getKey(), root->state.symbols); if (cachedValue && !std::get_if(&cachedValue->second)) { if (auto attrs = std::get_if>(&cachedValue->second)) { debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else throw TypeError("'%s' is not an attribute set", getAttrPathStr()); } } auto & v = forceValue(); if (v.type != tAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector attrs; for (auto & attr : *getValue().attrs) attrs.push_back(attr.name); std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { return (const string &) a < (const string &) b; }); if (root->db) cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; return attrs; } bool AttrCursor::isDerivation() { auto aType = maybeGetAttr("type"); return aType && aType->getString() == "derivation"; } StorePath AttrCursor::forceDerivation() { auto aDrvPath = getAttr(root->state.sDrvPath, true); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has been garbage-collected. So force it to be regenerated. */ aDrvPath->forceValue(); if (!root->state.store->isValidPath(drvPath)) throw Error("don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); } return drvPath; } }