From 1722ae6ecee54e14164d215ba3d767ea6c352fc3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jun 2020 05:41:18 +0000 Subject: [PATCH 001/114] Pull out PathReferences super class --- src/libstore/path-info.hh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index f5dee00a6..a980e1243 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -20,12 +20,17 @@ namespace nix { class Store; -struct ValidPathInfo +template +struct PathReferences +{ + std::set references; +}; + +struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only From 71e4c9c505f2418084643c1a68da5c89b82038dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jun 2020 22:46:27 +0000 Subject: [PATCH 002/114] WIP: store separate `hasValidPath` bool --- src/libstore/binary-cache-store.cc | 3 +- src/libstore/build.cc | 14 ++++---- src/libstore/daemon.cc | 10 +++--- src/libstore/export-import.cc | 4 +-- src/libstore/legacy-ssh-store.cc | 7 ++-- src/libstore/local-store.cc | 23 +++++++------ src/libstore/misc.cc | 5 ++- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/path-info.hh | 52 +++++++++++++++++++++++++++++ src/libstore/remote-store.cc | 11 +++--- src/libstore/store-api.cc | 29 ++++++++++------ src/libstore/store-api.hh | 11 ------ src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 ++--- src/nix/make-content-addressable.cc | 2 +- src/nix/sigs.cc | 3 +- 17 files changed, 119 insertions(+), 69 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..7167ec900 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -125,8 +125,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c25897f8..c01b2ddaf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3834,7 +3834,7 @@ void DerivationGoal::registerOutputs() ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = std::move(references); + info.setReferencesPossiblyToSelf(std::move(references)); info.deriver = drvPath; info.ultimate = true; info.ca = ca; @@ -3963,12 +3963,12 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -3997,7 +3997,7 @@ void DerivationGoal::checkOutputs(const std::map & outputs) auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -4466,8 +4466,7 @@ void SubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -4487,8 +4486,7 @@ void SubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 842aef20c..c6b70f6f3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -326,7 +326,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -601,7 +601,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - writeStorePaths(*store, to, i->second.references); + writeStorePaths(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -618,7 +618,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - writeStorePaths(*store, to, i.second.references); + writeStorePaths(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -647,7 +647,7 @@ static void performOp(TunnelLogger * logger, ref store, to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - writeStorePaths(*store, to, info->references); + writeStorePaths(*store, to, info->referencesPossiblyToSelf()); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -707,7 +707,7 @@ static void performOp(TunnelLogger * logger, ref store, if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths(*store, from); + info.setReferencesPossiblyToSelf(readStorePaths(*store, from)); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..5a5c76f7f 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -62,7 +62,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) hashAndWriteSink << exportMagic << printStorePath(path); - writeStorePaths(*this, hashAndWriteSink, info->references); + writeStorePaths(*this, hashAndWriteSink, info->referencesPossiblyToSelf()); hashAndWriteSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -88,7 +88,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr acces //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths(*this, source); + info.setReferencesPossiblyToSelf(readStorePaths(*this, source)); auto deriver = readString(source); if (deriver != "") diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 5657aa593..f01e642a0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -103,11 +103,10 @@ struct LegacySSHStore : public Store auto info = std::make_shared(parseStorePath(p)); assert(path == info->path); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -140,7 +139,7 @@ struct LegacySSHStore : public Store << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -169,7 +168,7 @@ struct LegacySSHStore : public Store conn->to << exportMagic << printStorePath(info.path); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0dfbed9fc..02de3aa5e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -671,8 +671,10 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) { + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); + } return info; })); @@ -856,11 +858,13 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); - infos.insert_or_assign(path, SubstitutablePathInfo{ - info->deriver, + infos.insert_or_assign(path, SubstitutablePathInfo { info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -907,7 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -986,14 +990,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !refersToSelf)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.hasSelfReference) hashSink = std::make_unique(htSHA256); else hashSink = std::make_unique(htSHA256, std::string(info.path.hashPart())); @@ -1254,7 +1257,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr hashSink; - if (!info->ca || !info->references.count(info->path)) + if (!info->ca || !info->hasSelfReference) hashSink = std::make_unique(*info->narHash.type); else hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e68edb38c..5214a7bf4 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -61,8 +61,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, } else { for (auto & ref : info->references) - if (ref != path) - enqueue(printStorePath(ref)); + enqueue(printStorePath(ref)); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) @@ -268,7 +267,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.count(i)) + if (paths.count(i)) dfsVisit(i, &path); sorted.push_back(path); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 012dea6ea..c543f6ea2 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -198,7 +198,7 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a980e1243..27efe5ae9 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -24,8 +24,43 @@ template struct PathReferences { std::set references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); }; +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet references { references }; + if (hasSelfReference) + references.insert(self); + return references; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + struct ValidPathInfo : PathReferences { StorePath path; @@ -64,6 +99,7 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -80,6 +116,12 @@ struct ValidPathInfo : PathReferences /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf() const; + void insertReferencePossiblyToSelf(StorePath && ref); + void setReferencesPossiblyToSelf(StorePathSet && refs); + static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were @@ -101,4 +143,14 @@ struct ValidPathInfo : PathReferences }; typedef list ValidPathInfos; + + +struct SubstitutablePathInfo : PathReferences +{ + std::optional deriver; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7cc7a5fc..f84b62f2e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -326,7 +326,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(i, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i, std::move(info)); @@ -339,11 +339,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.setReferencesPossiblyToSelf(path, readStorePaths(*this, conn->from)); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -376,7 +377,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths(*this, conn->from); + info->setReferencesPossiblyToSelf(readStorePaths(*this, conn->from)); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -455,7 +456,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths(*this, conn->from); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..95b1c1c3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -715,7 +715,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -738,6 +738,20 @@ string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -748,7 +762,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -767,16 +781,11 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const auto caPath = std::visit(overloaded { [&](TextHash th) { + assert(!hasSelfReference); return store.makeTextPath(path.name(), th.hash, references); }, [&](FixedOutputHash fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); } }, *ca); @@ -810,7 +819,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 00b9c385c..420ffebbe 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -102,17 +102,6 @@ struct GCResults }; -struct SubstitutablePathInfo -{ - std::optional deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map SubstitutablePathInfos; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8b699f39b..45abe0405 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -58,7 +58,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 8ca5c9c8d..1cd974e41 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7d81bf54f..c4ca89c85 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -245,7 +245,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -328,7 +328,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -859,7 +859,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - writeStorePaths(*store, out, info->references); + writeStorePaths(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -949,7 +949,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (deriver != "") info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths(*store, in); + info.setReferencesPossiblyToSelf(readStorePaths(*store, in)); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..5267948ee 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -79,7 +79,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.hasSelfReference = std::move(hasSelfReference); info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..a40975982 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -65,7 +65,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) From a9c0ea30bf81a42dfb7ccca04bb98649a6c34d07 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 17:59:27 +0000 Subject: [PATCH 003/114] Backport fix from #3754 branch --- src/libstore/path-info.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 27efe5ae9..a67c36bb6 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -36,10 +36,10 @@ struct PathReferences template StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const { - StorePathSet references { references }; + StorePathSet refs { references }; if (hasSelfReference) - references.insert(self); - return references; + refs.insert(self); + return refs; } template From 70ed47c1cb9d04a5a350cac664921a194d93d329 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:21:46 +0000 Subject: [PATCH 004/114] Fix some things in remote store --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f84b62f2e..912a3b70c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -445,7 +445,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + writeStorePaths(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -464,7 +464,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + writeStorePaths(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; From 66834068432d316ee558717765851835ceec2dcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jun 2020 19:58:31 +0000 Subject: [PATCH 005/114] Fix nar info parsing --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ef04bc859..0796de466 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -57,7 +57,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") From e61061c88e0dfcce9329ea9f0b041a35270dfa1a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 23:17:11 +0000 Subject: [PATCH 006/114] Remove stray tabs --- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d7aef3ea5..273455bae 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -386,7 +386,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a28386ef..0fee5559f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -913,17 +913,17 @@ string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return PathReferences::referencesPossiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const From f8d562c0a7cef27c65d3cff96ad8ef384f05b331 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Oct 2020 13:52:20 +0000 Subject: [PATCH 007/114] Use PathReferences more widely --- perl/lib/Nix/Store.xs | 10 +- src/libexpr/primops.cc | 16 ++- src/libexpr/value-to-xml.hh | 2 +- src/libfetchers/fetchers.cc | 8 +- src/libfetchers/tarball.cc | 16 ++- src/libstore/binary-cache-store.cc | 43 ++++-- src/libstore/build.cc | 30 ++-- src/libstore/content-address.cc | 24 +++- src/libstore/content-address.hh | 102 +++++++++++++- src/libstore/derivations.cc | 4 +- src/libstore/derivations.hh | 4 +- src/libstore/local-store.cc | 30 +++- src/libstore/nar-info.hh | 3 + src/libstore/path-info.hh | 47 +------ src/libstore/path.hh | 5 +- src/libstore/store-api.cc | 171 +++++++++++++++-------- src/libstore/store-api.hh | 14 +- src/libutil/args.cc | 2 +- src/libutil/error.hh | 3 +- src/libutil/fmt.hh | 2 +- src/libutil/tests/logging.cc | 2 +- src/libutil/types.hh | 1 + src/nix-prefetch-url/nix-prefetch-url.cc | 10 +- src/nix-store/nix-store.cc | 12 +- src/nix/add-to-store.cc | 16 ++- src/nix/bundle.cc | 2 +- src/nix/make-content-addressable.cc | 38 ++--- src/nix/profile.cc | 13 +- src/nix/verify.cc | 6 +- 29 files changed, 431 insertions(+), 205 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..ea8bbaf34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { @@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..c74b67658 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..67bb77d3e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..b3ee84810 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,14 +71,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index d592f16dd..2d92e1c50 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; auto source = StringSource { s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97a832c6b..12ce6f2ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.references, - refs.hasSelfReference), + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, narHashAndSize.first, }; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - static_cast &>(newInfo0) = refs; assert(newInfo0.ca); return newInfo0; @@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..e15d76bd7 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,20 @@ #include #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; + struct TextHash { Hash hash; }; @@ -41,10 +47,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod); +/* + * References set + */ + +template +struct PathReferences +{ + std::set references; + bool hasSelfReference = false; + + bool operator == (const PathReferences & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); +}; + +template +StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template +void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template +void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..925a78083 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,8 +27,8 @@ std::optional DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..be19aa300 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -130,8 +130,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 18545f659..e6b02cce6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // recompute store path so that we can use a different store root if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 509f100d7..8c4791ac0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,47 +13,6 @@ namespace nix { class Store; -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct SubstitutablePathInfo : PathReferences { @@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences { StorePath path; std::optional deriver; - // TODO document this Hash narHash; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences void sign(const Store & store, const SecretKey & secretKey); + std::optional fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..5f239ceb6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set StorePathSet; typedef std::vector StorePaths; typedef std::map OutputPathMap; -typedef std::map> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7041edbe5..5d63b8e3c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" @@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -229,14 +228,24 @@ std::pair Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePath, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref(info)); @@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -737,7 +753,8 @@ void copyStorePath(ref srcStore, ref dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -799,7 +816,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -826,7 +844,8 @@ std::map copyPaths(ref srcStore, ref dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..e6a6053a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -170,6 +170,8 @@ struct BuildResult } }; +typedef std::map> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -313,17 +315,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..453fe60f9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -244,7 +244,7 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..260ed3cf8 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -201,9 +201,8 @@ public: template SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..11dbef9db 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -103,7 +103,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..d33bd7c1f 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -370,7 +370,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 55d02bcf9..6c4c5ab74 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include #include +#include #include #include diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..99cc0cdec 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -161,8 +161,14 @@ static int _main(int argc, char * * argv) std::optional storePath; if (args.size() == 2) { expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = *expectedHash; else diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9092dbd80..7981bbbdd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..86616d66b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..510df7504 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -91,7 +91,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7737f6d91..7695c98f8 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..41a4857fc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -130,12 +130,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..d189a2fd3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); From 39c11c5c01de9c18bf1b0bc3928fc28393fd0ca9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 03:30:14 +0000 Subject: [PATCH 008/114] Organize content-address.hh a bit better --- src/libstore/content-address.hh | 53 ++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e15d76bd7..126244ab5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -7,7 +7,7 @@ namespace nix { /* - * Mini content address + * Content addressing method */ enum struct FileIngestionMethod : uint8_t { @@ -15,6 +15,34 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + +/* + * Mini content address + */ struct TextHash { Hash hash; @@ -43,10 +71,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional ca); @@ -57,25 +81,6 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; - -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); - -std::string renderContentAddressMethod(ContentAddressMethod caMethod); - /* * References set */ From 10e81bf871551901ff0383bdede0f79325e93867 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 15 Oct 2020 02:21:28 +0000 Subject: [PATCH 009/114] Fix conditions for ca-references --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e6b02cce6..36ef7acf7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1071,7 +1071,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && info.hasSelfReference)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative(*info.ca) && !info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash From 2c21cb672043fcf3c3fd19f89618b37693c0dc62 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 22:40:40 +0000 Subject: [PATCH 010/114] Fill in missing comparison operators for content addresses --- src/libstore/content-address.hh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 126244ab5..acdb4f023 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" #include "path.hh" +#include "comparator.hh" namespace nix { @@ -46,6 +47,8 @@ std::string renderContentAddressMethod(ContentAddressMethod caMethod); struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -53,6 +56,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -91,17 +96,13 @@ struct PathReferences std::set references; bool hasSelfReference = false; - bool operator == (const PathReferences & other) const - { - return references == other.references - && hasSelfReference == other.hasSelfReference; - } - /* Functions to view references + hasSelfReference as one set, mainly for compatibility's sake. */ StorePathSet referencesPossiblyToSelf(const Ref & self) const; void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); }; template @@ -142,11 +143,15 @@ void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set struct TextInfo : TextHash { // References for the paths, self references disallowed StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); }; struct FixedOutputInfo : FixedOutputHash { // References for the paths PathReferences references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< @@ -160,17 +165,7 @@ struct StorePathDescriptor { std::string name; ContentAddressWithReferences info; - bool operator == (const StorePathDescriptor & other) const - { - return name == other.name; - // FIXME second field - } - - bool operator < (const StorePathDescriptor & other) const - { - return name < other.name; - // FIXME second field - } + GENERATE_CMP(StorePathDescriptor, me->name, me->info); }; } From 13c669105ca93d28ca1a78321f07fd4ddbb445b1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 19 Apr 2022 22:25:21 +0000 Subject: [PATCH 011/114] Slight cleanups --- src/libstore/content-address.cc | 2 +- src/libstore/content-address.hh | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e4ba855d5..2e6c435ce 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -10,7 +10,7 @@ std::string FixedOutputHash::printMethodAlgo() const } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index acdb4f023..a275800f9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -11,17 +11,16 @@ namespace nix { * Content addressing method */ +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ +struct TextHashMethod : std::monostate { }; + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; - struct FixedOutputHashMethod { FileIngestionMethod fileIngestionMethod; HashType hashType; @@ -29,9 +28,13 @@ struct FixedOutputHashMethod { /* Compute the prefix to the hash algorithm which indicates how the files were ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); +std::string makeFileIngestionPrefix(FileIngestionMethod m); +/* Just the type of a content address. Combine with the hash itself, and we + have a `ContentAddress` as defined below. Combine that, in turn, with info + on references, and we have `ContentAddressWithReferences`, as defined + further below. */ typedef std::variant< TextHashMethod, FixedOutputHashMethod @@ -86,6 +89,7 @@ std::optional parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); + /* * References set */ From c7188c96f6b1cc4b09b4a4b7cc64df2370f24feb Mon Sep 17 00:00:00 2001 From: Ezra Singh Date: Wed, 24 Aug 2022 13:07:32 -0400 Subject: [PATCH 012/114] Update install-systemd-multi-user.sh --- scripts/install-systemd-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 62397127a..1a564d91d 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -87,7 +87,7 @@ poly_configure_nix_daemon_service() { task "Setting up the nix-daemon systemd service" _sudo "to create the nix-daemon tmpfiles config" \ - ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST + ln -sfn /nix/var/nix/profiles/default$TMPFILES_SRC $TMPFILES_DEST _sudo "to run systemd-tmpfiles once to pick that path up" \ systemd-tmpfiles --create --prefix=/nix/var/nix From 8623143921f8683b88d46aaebe9f707e5b9a912b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 11:18:14 -0500 Subject: [PATCH 013/114] Make formatting consistent --- src/libstore/content-address.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index a275800f9..6be4be4c5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -38,7 +38,7 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); typedef std::variant< TextHashMethod, FixedOutputHashMethod - > ContentAddressMethod; +> ContentAddressMethod; ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); From 6a168254ce068c067259c913ee7d6ee2e0d1dc7e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:24:20 -0500 Subject: [PATCH 014/114] Use named field initialization for references --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 10 ++++++++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/store-api.cc | 22 +++++++++++---------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 15 files changed, 34 insertions(+), 26 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9cb078660..3ccd3c722 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8e9e5630d..8a19eab8f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1249,7 +1249,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * .method = method, .hash = h, }, - {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 4181f0b7d..560a086f0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -235,7 +235,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 735d9fc93..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index d52d19797..155b86cc4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -79,7 +79,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - {}, + .references = {}, }, }, hashString(htSHA256, sink.s), diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3bbf4c8ac..aac5e7b88 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - { + .references = { .references = references, .hasSelfReference = false, }, @@ -433,7 +433,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 110a6a301..3d8299bbf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2482,7 +2482,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - rewriteRefs(), + .references = rewriteRefs(), }, }, Hash::dummy, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 2e6c435ce..3b8a773b7 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -154,10 +154,16 @@ Hash getContentAddressHash(const ContentAddress & ca) ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { - return TextInfo { h, {}}; + return TextInfo { + h, + .references = {}, + }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { - return FixedOutputInfo { h, {}}; + return FixedOutputInfo { + h, + .references = {}, + }; }, }, ca); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a8f060768..9f3a6db24 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1426,7 +1426,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - { + .references = { .references = references, .hasSelfReference = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 9655a0555..d6b6e87c9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -55,7 +55,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - std::move(refs), + .references = std::move(refs), }, }, Hash::dummy, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 592afebd8..4b89465e7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -442,7 +442,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - {}, + .references = {}, }, }, narHash, @@ -1270,16 +1270,18 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c return StorePathDescriptor { .name = std::string { path.name() }, .info = std::visit(overloaded { - [&](const TextHash & th) { - TextInfo info { th }; + [&](const TextHash & th) -> ContentAddressWithReferences { assert(!hasSelfReference); - info.references = references; - return ContentAddressWithReferences { info }; + return TextInfo { + th, + .references = references, + }; }, - [&](const FixedOutputHash & foh) { - FixedOutputInfo info { foh }; - info.references = static_cast>(*this); - return ContentAddressWithReferences { info }; + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + return FixedOutputInfo { + foh, + .references = static_cast>(*this), + }; }, }, *ca), }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75fa08551..5cb5aa53a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index c2494dc9f..0b58818c3 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - {}, + .references = {}, }, }, narHash, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index aa302efc1..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 024849e3b..614a37eba 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,7 +205,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - { references }, + .references = { references }, }, }, narHash, From 9cfa78e58a92b4bf034867bc1296a200bdc3f12a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 12:26:15 -0500 Subject: [PATCH 015/114] Optimize `ValidPathInfo` construction a bit better --- src/libstore/store-api.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4b89465e7..cd48d616b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1340,13 +1340,13 @@ ValidPathInfo::ValidPathInfo( , narHash(narHash) { std::visit(overloaded { - [this](const TextInfo & ti) { - this->references = ti.references; - this->ca = TextHash { std::move(ti) }; + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); }, - [this](const FixedOutputInfo & foi) { - *(static_cast *>(this)) = foi.references; - this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + [this](FixedOutputInfo && foi) { + *(static_cast *>(this)) = std::move(foi.references); + this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } From 46e942ff9e65755689ee72f93846d7118e1b8d45 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 15:36:05 -0500 Subject: [PATCH 016/114] Do big rename to clean up code - `PathReferences` -> `References` - `PathReferences` -> `StoreReference` - `references` -> `others` - `hasSelfReference` -> `self` And get rid of silly subclassing --- src/libexpr/primops.cc | 7 ++- src/libstore/binary-cache-store.cc | 13 ++-- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 6 +- src/libstore/content-address.hh | 46 +------------- src/libstore/daemon.cc | 4 +- src/libstore/local-store.cc | 18 +++--- src/libstore/make-content-addressed.cc | 8 +-- src/libstore/misc.cc | 16 ++--- src/libstore/path-info.hh | 7 ++- src/libstore/remote-store.cc | 4 +- src/libstore/store-api.cc | 36 +++++------ src/libutil/reference-set.hh | 68 +++++++++++++++++++++ src/nix/profile.cc | 6 +- src/nix/sigs.cc | 3 +- src/nix/why-depends.cc | 2 +- 16 files changed, 146 insertions(+), 112 deletions(-) create mode 100644 src/libutil/reference-set.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a19eab8f..0113659d1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1535,7 +1535,8 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + // FIXME: Are self references becoming non-self references OK? + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1971,7 +1972,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->references; + refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } @@ -2010,7 +2011,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index aac5e7b88..aa5aafdbf 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,8 +180,9 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. */ - for (auto & ref : info.references) + reads, but typically they'll already be cached. Note that + self-references are always valid. */ + for (auto & ref : info.references.others) try { queryPathInfo(ref); } catch (InvalidPath &) { @@ -314,8 +315,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .hash = nar.first, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, @@ -434,8 +435,8 @@ StorePath BinaryCacheStore::addToStore( .hash = h, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3d8299bbf..ff24bd088 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2421,26 +2421,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> PathReferences { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - PathReferences res { - .hasSelfReference = false, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.hasSelfReference = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.references.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.references.insert(r); + res.others.insert(r); } } return res; @@ -2523,7 +2523,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - static_cast &>(newInfo0) = rewriteRefs(); + newInfo0.references = rewriteRefs(); return newInfo0; }, diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 307183505..36b0ea7a7 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -128,7 +128,7 @@ void PathSubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { + if (info->isContentAddressed(*sub) && info->references.empty()) { auto info2 = std::make_shared(*info); info2->path = storePath; info = info2; @@ -165,7 +165,7 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references) + for (auto & i : info->references.others) addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -187,7 +187,7 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references) + for (auto & i : info->references.others) assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 6be4be4c5..f8a4d5370 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,6 +4,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "reference-set.hh" namespace nix { @@ -94,48 +95,7 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -template -struct PathReferences -{ - std::set references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(PathReferences, me->references, me->hasSelfReference); -}; - -template -StorePathSet PathReferences::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template -void PathReferences::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template -void PathReferences::setReferencesPossiblyToSelf(const Ref & self, std::set && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} +typedef References StoreReferences; /* * Full content address @@ -153,7 +113,7 @@ struct TextInfo : TextHash { struct FixedOutputInfo : FixedOutputHash { // References for the paths - PathReferences references; + StoreReferences references; GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6407e575a..605f871fc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9f3a6db24..b32953f3f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1157,11 +1157,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast( std::shared_ptr(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->references, - info->hasSelfReference, - info->deriver, - narInfo ? narInfo->fileSize : 0, - info->narSize, + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { @@ -1228,7 +1227,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references; + return i == infos.end() ? StorePathSet() : i->second.references.others; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1427,8 +1426,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .hash = hash, }, .references = { - .references = references, - .hasSelfReference = false, + .others = references, + .self = false, }, }, }; @@ -1526,7 +1525,8 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - info.references = references; + // No self reference allowed with text-hashing + info.references.others = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index d6b6e87c9..5d7945eb1 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,15 +27,15 @@ std::map makeContentAddressed( StringMap rewrites; - PathReferences refs; - refs.hasSelfReference = oldInfo->hasSelfReference; - for (auto & ref : oldInfo->references) { + StoreReferences refs; + refs.self = oldInfo->references.self; + for (auto & ref : oldInfo->references.others) { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) { rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index e0bb1fab0..87f85c3cc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto& ref : referrers) + for (auto & ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto& i : queryValidDerivers(path)) + for (auto & i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,11 +40,11 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto& ref : info->references) + for (auto & ref : info->references.others) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); @@ -223,7 +223,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references) + for (auto & ref : info->second.references.others) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +241,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references; + return queryPathInfo(path)->references.others; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +297,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 89886873a..9254835b7 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,20 +14,22 @@ namespace nix { class Store; -struct SubstitutablePathInfo : PathReferences +struct SubstitutablePathInfo { std::optional deriver; + StoreReferences references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; -struct ValidPathInfo : PathReferences +struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; + StoreReferences references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -61,7 +63,6 @@ struct ValidPathInfo : PathReferences return path == i.path && narHash == i.narHash - && hasSelfReference == i.hasSelfReference && references == i.references; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 689ad3fbe..1f8098b85 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -426,7 +426,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cd48d616b..5490df292 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -169,13 +169,13 @@ StorePath Store::makeOutputPath(std::string_view id, static std::string makeType( const Store & store, std::string && type, - const PathReferences & references) + const StoreReferences & references) { - for (auto & i : references.references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (references.hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } @@ -185,8 +185,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(info.references.references.size() == 0); - assert(!info.references.hasSelfReference); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" @@ -201,7 +200,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", PathReferences { info.references }), + makeType(*this, "text", StoreReferences { info.references }), info.hash, name); } @@ -311,7 +310,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references; + return info.references.others; }, [&](const StorePath & path) { @@ -816,7 +815,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->references) + for (auto & j : info->referencesPossiblyToSelf()) s += printStorePath(j) + "\n"; } @@ -878,7 +877,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1231,17 +1230,17 @@ std::string showPaths(const PathSet & paths) StorePathSet ValidPathInfo::referencesPossiblyToSelf() const { - return PathReferences::referencesPossiblyToSelf(path); + return references.possiblyToSelf(path); } void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) { - return PathReferences::insertReferencePossiblyToSelf(path, std::move(ref)); + return references.insertPossiblyToSelf(path, std::move(ref)); } void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) { - return PathReferences::setReferencesPossiblyToSelf(path, std::move(refs)); + return references.setPossiblyToSelf(path, std::move(refs)); } std::string ValidPathInfo::fingerprint(const Store & store) const @@ -1271,16 +1270,16 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!hasSelfReference); + assert(!references.self); return TextInfo { th, - .references = references, + .references = references.others, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { return FixedOutputInfo { foh, - .references = static_cast>(*this), + .references = references, }; }, }, *ca), @@ -1341,11 +1340,14 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = std::move(ti.references); + this->references = { + .others = std::move(ti.references), + .self = false, + }; this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - *(static_cast *>(this)) = std::move(foi.references); + this->references = std::move(foi.references); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh new file mode 100644 index 000000000..ac4a9994e --- /dev/null +++ b/src/libutil/reference-set.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "comparator.hh" + +#include + +namespace nix { + +template +struct References +{ + std::set others; + bool self = false; + + bool empty() const; + size_t size() const; + + /* Functions to view references + self as one set, mainly for + compatibility's sake. */ + std::set possiblyToSelf(const Ref & self) const; + void insertPossiblyToSelf(const Ref & self, Ref && ref); + void setPossiblyToSelf(const Ref & self, std::set && refs); + + GENERATE_CMP(References, me->others, me->self); +}; + +template +bool References::empty() const +{ + return !self && others.empty(); +} + +template +size_t References::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +template +std::set References::possiblyToSelf(const Ref & selfRef) const +{ + std::set refs { others }; + if (self) + refs.insert(selfRef); + return refs; +} + +template +void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) +{ + if (ref == selfRef) + self = true; + else + others.insert(std::move(ref)); +} + +template +void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) +{ + if (refs.count(selfRef)) { + self = true; + refs.erase(selfRef); + } + + others = refs; +} + +} diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 614a37eba..8a0f06435 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -205,12 +205,14 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { references }, + .references = { + .others = std::move(references), + .self = false, + }, }, }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); StringSource source(sink.s); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index a08314a25..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -63,8 +63,7 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references || - info->hasSelfReference != info2->hasSelfReference) + info->references != info2->references) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..33cd13600 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references, + .refs = store->queryPathInfo(path)->references.others, .dist = path == dependencyPath ? 0 : inf }); From 91617f80ec03ff4580a656310959ce2e31e0d177 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 6 Jan 2023 16:00:10 -0500 Subject: [PATCH 017/114] Fix perl bindings --- perl/lib/Nix/Store.xs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3ccd3c722..bdb4fa655 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From 2e7be46e73293f729358eefc5b464dcb7e2d76bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 13 Jan 2023 15:06:07 -0500 Subject: [PATCH 018/114] Move new `ValidPathInfo` methods to path-info.cc We'll move the old ones separately, so as not to clutter the diff. --- src/libstore/path-info.cc | 15 +++++++++++++++ src/libstore/store-api.cc | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 003685604..cb3077c61 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,21 @@ namespace nix { +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return references.possiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return references.insertPossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return references.setPossiblyToSelf(path, std::move(refs)); +} + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5490df292..a4e98d66b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1228,21 +1228,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) From b3d91239ae9f21a60057b278ceeff663fb786246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 14 Jan 2023 16:38:43 -0500 Subject: [PATCH 019/114] Make `ValidPathInfo` have plain `StorePathSet` references like before This change can wait for another PR. --- perl/lib/Nix/Store.xs | 4 +- src/libexpr/primops.cc | 5 +- src/libstore/binary-cache-store.cc | 8 +-- src/libstore/build/local-derivation-goal.cc | 11 ++-- src/libstore/build/substitution-goal.cc | 10 +-- src/libstore/content-address.cc | 10 +++ src/libstore/content-address.hh | 11 +++- src/libstore/daemon.cc | 8 +-- src/libstore/export-import.cc | 4 +- src/libstore/legacy-ssh-store.cc | 6 +- src/libstore/local-store.cc | 10 ++- src/libstore/make-content-addressed.cc | 5 +- src/libstore/misc.cc | 11 ++-- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 48 ++++++--------- src/libstore/path-info.hh | 11 +--- src/libstore/remote-store.cc | 13 ++-- src/libstore/store-api.cc | 14 +++-- src/libutil/reference-set.hh | 68 --------------------- src/nix-store/dotgraph.cc | 2 +- src/nix-store/graphml.cc | 2 +- src/nix-store/nix-store.cc | 8 +-- src/nix/why-depends.cc | 2 +- 24 files changed, 109 insertions(+), 166 deletions(-) delete mode 100644 src/libutil/reference-set.hh diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bdb4fa655..f19fb20bf 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -69,7 +69,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf()) + for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->referencesPossiblyToSelf()) + for (auto & i : info->references) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ae573cf4d..3b32625b1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1544,8 +1544,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - // FIXME: Are self references becoming non-self references OK? - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1980,7 +1979,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 087b37655..ac41add2c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. Note that - self-references are always valid. */ - for (auto & ref : info.references.others) + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) try { - queryPathInfo(ref); + if (ref != info.path) + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d96858fc0..bb4f92989 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2523,7 +2523,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - newInfo0.references = rewriteRefs(); + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2774,12 +2777,12 @@ void LocalDerivationGoal::checkOutputs(const std::mapsecond.narSize; - for (auto & ref : i->second.referencesPossiblyToSelf()) + for (auto & ref : i->second.references) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) pathsLeft.push(ref); } } @@ -2819,7 +2822,7 @@ void LocalDerivationGoal::checkOutputs(const std::mapreferences.others) - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,8 +188,9 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references.others) - assert(worker.store.isValidPath(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 3b8a773b7..a98e34cb8 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -151,6 +151,16 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f8a4d5370..4a50bbee0 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,7 +4,6 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" -#include "reference-set.hh" namespace nix { @@ -95,7 +94,15 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -typedef References StoreReferences; +struct StoreReferences { + StorePathSet others; + bool self = false; + + bool empty() const; + size_t size() const; + + GENERATE_CMP(StoreReferences, me->self, me->others); +}; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 605f871fc..12596ba49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) + for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom {})); + info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4adf51573..9875da909 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); + worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(std::move(references)); + info.references = references; info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 6a694f034..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom {})); + info->references = worker_proto::read(*this, conn->from, Phantom {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b32953f3f..2d03d2d8b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,8 +938,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->insertReferencePossiblyToSelf( - parseStorePath(useQueryReferences.getStr(0))); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1206,7 +1205,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.referencesPossiblyToSelf()) + for (auto & j : i.references) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1227,7 +1226,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references.others; + return i == infos.end() ? StorePathSet() : i->second.references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1525,8 +1524,7 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - // No self reference allowed with text-hashing - info.references.others = references; + info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 5d7945eb1..09f615439 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,8 +28,9 @@ std::map makeContentAddressed( StringMap rewrites; StoreReferences refs; - refs.self = oldInfo->references.self; - for (auto & ref : oldInfo->references.others) { + for (auto & ref : oldInfo->references) { + if (ref == path) + refs.self = true; auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 70e97569a..da96dcebc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,8 +40,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references.others) - res.insert(ref); + for (auto & ref : info->references) + if (ref != path) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -223,7 +224,7 @@ void Store::queryMissing(const std::vector & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references.others) + for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -241,7 +242,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references.others; + return queryPathInfo(path)->references; } catch (InvalidPath &) { return StorePathSet(); } @@ -297,7 +298,7 @@ std::map drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8d4a21daf..3e0689534 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) - narInfo->insertReferencePossiblyToSelf(StorePath(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index f54e8f1fc..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - insertReferencePossiblyToSelf(StorePath(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2972c0bbe..93f91e702 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); + + concatStringsSep(",", store.printStorePathSet(references)); } @@ -30,16 +30,25 @@ std::optional ValidPathInfo::fullStorePathDescriptorOpt() c .name = std::string { path.name() }, .info = std::visit(overloaded { [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!references.self); + assert(references.count(path) == 0); return TextInfo { th, - .references = references.others, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } return FixedOutputInfo { foh, - .references = references, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, }; }, }, *ca), @@ -85,7 +94,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : referencesPossiblyToSelf()) + for (auto & r : references) refs.push_back(std::string(r.to_string())); return refs; } @@ -100,36 +109,19 @@ ValidPathInfo::ValidPathInfo( { std::visit(overloaded { [this](TextInfo && ti) { - this->references = { - .others = std::move(ti.references), - .self = false, - }; + this->references = std::move(ti.references); this->ca = std::move((TextHash &&) ti); }, [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references); + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, }, std::move(info.info)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - - ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -141,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom {})); + info.references = worker_proto::read(store, source, Phantom {}); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -162,7 +154,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, referencesPossiblyToSelf()); + worker_proto::write(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 9254835b7..476df79c2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,19 +17,20 @@ class Store; struct SubstitutablePathInfo { std::optional deriver; - StoreReferences references; + StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map SubstitutablePathInfos; + struct ValidPathInfo { StorePath path; std::optional deriver; Hash narHash; - StoreReferences references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -81,12 +82,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8ea126c65..ff57a77ca 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,12 +421,11 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); - SubstitutablePathInfo & info { infos[path] }; + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom {})); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -634,7 +633,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -645,7 +644,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); - assert(importedPaths.empty() == 0); // doesn't include possible self reference + assert(importedPaths.size() <= 1); } else { @@ -653,7 +652,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9446ad132..c39e50d14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,7 +200,10 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { info.references }), + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), info.hash, name); } @@ -310,7 +313,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references.others; + return info.references; }, [&](const StorePath & path) { @@ -815,7 +818,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->referencesPossiblyToSelf()) + for (auto & j : info->references) s += printStorePath(j) + "\n"; } @@ -877,7 +880,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1205,7 +1208,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.insertReferencePossiblyToSelf(store.parseStorePath(s)); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional(std::move(info)); @@ -1228,6 +1231,7 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh deleted file mode 100644 index ac4a9994e..000000000 --- a/src/libutil/reference-set.hh +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include "comparator.hh" - -#include - -namespace nix { - -template -struct References -{ - std::set others; - bool self = false; - - bool empty() const; - size_t size() const; - - /* Functions to view references + self as one set, mainly for - compatibility's sake. */ - std::set possiblyToSelf(const Ref & self) const; - void insertPossiblyToSelf(const Ref & self, Ref && ref); - void setPossiblyToSelf(const Ref & self, std::set && refs); - - GENERATE_CMP(References, me->others, me->self); -}; - -template -bool References::empty() const -{ - return !self && others.empty(); -} - -template -size_t References::size() const -{ - return (self ? 1 : 0) + others.size(); -} - -template -std::set References::possiblyToSelf(const Ref & selfRef) const -{ - std::set refs { others }; - if (self) - refs.insert(selfRef); - return refs; -} - -template -void References::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) -{ - if (ref == selfRef) - self = true; - else - others.insert(std::move(ref)); -} - -template -void References::setPossiblyToSelf(const Ref & selfRef, std::set && refs) -{ - if (refs.count(selfRef)) { - self = true; - refs.erase(selfRef); - } - - others = refs; -} - -} diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 36d774dca..577cadceb 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { + for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index d2eebca7a..425d61e53 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->referencesPossiblyToSelf()) { + for (auto & p : info->references) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5cb5aa53a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) + for (auto & p : store->queryPathInfo(j)->references) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->referencesPossiblyToSelf()); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom {})); + info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 33cd13600..76125e5e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references.others, + .refs = store->queryPathInfo(path)->references, .dist = path == dependencyPath ? 0 : inf }); From 4540e7b940ca56db821fe7c7d7d79fafa488f55e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 23 Jan 2023 12:58:11 -0500 Subject: [PATCH 020/114] Don't add `StorePathDescriptor` for now We don't need it yet, we can add it back later. --- src/libfetchers/tarball.cc | 14 ++--- src/libstore/binary-cache-store.cc | 50 +++++++--------- src/libstore/build/local-derivation-goal.cc | 14 ++--- src/libstore/build/substitution-goal.cc | 7 +-- src/libstore/content-address.hh | 7 --- src/libstore/local-store.cc | 35 +++++------ src/libstore/make-content-addressed.cc | 14 ++--- src/libstore/nar-info.hh | 4 +- src/libstore/path-info.cc | 64 ++++++++++----------- src/libstore/path-info.hh | 4 +- src/libstore/store-api.cc | 28 ++++----- src/libstore/store-api.hh | 2 +- src/nix/add-to-store.cc | 14 ++--- src/nix/profile.cc | 20 +++---- 14 files changed, 126 insertions(+), 151 deletions(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 155b86cc4..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,15 +72,13 @@ DownloadFileResult downloadFile( auto hash = hashString(htSHA256, res.data); ValidPathInfo info { *store, - { - .name = name, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, }, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ac41add2c..9058bb8b1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -307,17 +307,15 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = nar.first, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -427,17 +425,15 @@ StorePath BinaryCacheStore::addToStore( return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = FixedOutputInfo { - { - .method = method, - .hash = h, - }, - .references = { - .others = references, - .self = false, - }, + name, + FixedOutputInfo { + { + .method = method, + .hash = h, + }, + .references = { + .others = references, + .self = false, }, }, nar.first, @@ -465,12 +461,10 @@ StorePath BinaryCacheStore::addTextToStore( return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, - { - .name = std::string { name }, - .info = TextInfo { - { .hash = textHash }, - references, - }, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, }, nar.first, }; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bb4f92989..98f8cb061 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2475,15 +2475,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto got = caSink.finish().first; ValidPathInfo newInfo0 { worker.store, - { - .name = outputPathName(drv->name, outputName), - .info = FixedOutputInfo { - { - .method = outputHash.method, - .hash = got, - }, - .references = rewriteRefs(), + outputPathName(drv->name, outputName), + FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, }, + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 994cb4ac2..87fed495c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,10 +95,9 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA({ - .name = std::string { storePath.name() }, - .info = caWithoutRefs(*ca), - }); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + caWithoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 4a50bbee0..c49ab269f 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -132,11 +132,4 @@ typedef std::variant< ContentAddressWithReferences caWithoutRefs(const ContentAddress &); -struct StorePathDescriptor { - std::string name; - ContentAddressWithReferences info; - - GENERATE_CMP(StorePathDescriptor, me->name, me->info); -}; - } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d03d2d8b..e55ccab84 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1136,10 +1136,9 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA({ - .name = std::string { path.first.name() }, - .info = caWithoutRefs(*path.second), - }); + subPath = makeFixedOutputPathFromCA( + path.first.name(), + caWithoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1417,21 +1416,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto desc = StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = { - .others = references, - .self = false, - }, + ContentAddressWithReferences desc = FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + .self = false, }, }; - auto dstPath = makeFixedOutputPathFromCA(desc); + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1475,7 +1471,12 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { *this, std::move(desc), narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; registerValidPath(info); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 09f615439..3ee64c77a 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -49,15 +49,13 @@ std::map makeContentAddressed( ValidPathInfo info { dstStore, - StorePathDescriptor { - .name = std::string { path.name() }, - .info = FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, - .references = std::move(refs), + path.name(), + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, }, + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index f1e3aabbd..a4dccb397 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -16,8 +16,8 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) - : ValidPathInfo(store, std::move(ca), narHash) + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 93f91e702..5944afd06 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,48 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::fullStorePathDescriptorOpt() const +std::optional ValidPathInfo::contentAddressWithReferenences() const { if (! ca) return std::nullopt; - return StorePathDescriptor { - .name = std::string { path.name() }, - .info = std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(references.count(path) == 0); - return TextInfo { - th, - .references = references, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return FixedOutputInfo { - foh, - .references = { - .others = std::move(refs), - .self = hasSelfReference, - }, - }; - }, - }, *ca), - }; + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca); } bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = fullStorePathDescriptorOpt(); + auto fullCaOpt = contentAddressWithReferenences(); if (! fullCaOpt) return false; - auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); bool res = caPath == path; @@ -102,9 +99,10 @@ Strings ValidPathInfo::shortRefs() const ValidPathInfo::ValidPathInfo( const Store & store, - StorePathDescriptor && info, + std::string_view name, + ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(info)) + : path(store.makeFixedOutputPathFromCA(name, ca)) , narHash(narHash) { std::visit(overloaded { @@ -118,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(info.info)); + }, std::move(ca)); } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 476df79c2..663d94540 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -77,7 +77,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional fullStorePathDescriptorOpt() const; + std::optional contentAddressWithReferenences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -100,7 +100,7 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const Store & store, - StorePathDescriptor && ca, Hash narHash); + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c39e50d14..3c0c26706 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -209,17 +209,17 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { [&](const TextInfo & ti) { - return makeTextPath(desc.name, ti); + return makeTextPath(name, ti); }, [&](const FixedOutputInfo & foi) { - return makeFixedOutputPath(desc.name, foi); + return makeFixedOutputPath(name, foi); } - }, desc.info); + }, ca); } @@ -437,15 +437,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, ValidPathInfo info { *this, - StorePathDescriptor { - std::string { name }, - FixedOutputInfo { - { - .method = method, - .hash = hash, - }, - .references = {}, + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, }, + .references = {}, }, narHash, }; @@ -997,7 +995,8 @@ void copyStorePath( if (info->ca && info->references.empty()) { auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( - info->fullStorePathDescriptorOpt().value()); + info->path.name(), + info->contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1109,8 @@ std::map copyPaths( auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( - currentPathInfo.fullStorePathDescriptorOpt().value()); + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferenences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d77aea338..2d252db84 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,7 +216,7 @@ public: StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 0b58818c3..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -43,15 +43,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info { *store, - StorePathDescriptor { - .name = *namePart, - .info = FixedOutputInfo { - { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, - .references = {}, + std::move(*namePart), + FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), }, + .references = {}, }, narHash, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index aac8e5c81..345505532 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,17 +200,15 @@ struct ProfileManifest ValidPathInfo info { *store, - StorePathDescriptor { - "profile", - FixedOutputInfo { - { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, - .references = { - .others = std::move(references), - .self = false, - }, + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + .self = false, }, }, narHash, From 974a983351283a644228b10731e4f9d2ff01e533 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 30 Jan 2023 09:59:55 -0500 Subject: [PATCH 021/114] Shrink diff in two places Stuff crept in there. --- src/libstore/content-address.cc | 3 ++- src/libstore/make-content-addressed.cc | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a98e34cb8..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..ff9f5cdaa 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,12 +31,14 @@ std::map makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.others.insert(std::move(replacement)); + } } } From 0983a0bd3050d02659f7c58555b8cbcfffed2c3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:04:28 -0500 Subject: [PATCH 022/114] Shrink diff in one place --- src/libstore/make-content-addressed.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index ff9f5cdaa..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -35,10 +35,9 @@ std::map makeContentAddressed( auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? - if (replacement != ref) { + if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - refs.others.insert(std::move(replacement)); - } + refs.others.insert(std::move(replacement)); } } From db759b1bc23c64b2aa6bdd0c5444a6d864488671 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:07:54 -0500 Subject: [PATCH 023/114] Undo style change `&` without space before is far more common on this codebase than I thought, so it is not worth changing just this one file. Maybe we will adopt a formatter someday but until then this is fine. --- src/libstore/misc.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1c187535d..b28768459 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); From 59d3175649a6bbdde76d1dfcf476c11392add827 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 10:09:25 -0500 Subject: [PATCH 024/114] Put back TODO I don't think the `narHash` is in need of documentation more than the other undocumented fields, but regardless this change has nothing to do with that field and so we should leave the comment as is. --- src/libstore/path-info.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..35aced472 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; From ee9eb83a842eb97d0180fd9d349d30ff27fdb485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Feb 2023 11:25:56 -0500 Subject: [PATCH 025/114] Remove some designated initializers With the switch to C++20, the rules became more strict, and we can no longer initialize base classes. Make them comments instead. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index f19fb20bf..314183383 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -299,7 +299,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8b54c4477..4e2f92276 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e194462e4..69395ad3d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -240,7 +240,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - .references = {} + /* .references = */ {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..dae4998f9 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - .references = {}, + /* .references = */ {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..b6f72bb1f 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..5617e2c42 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - .references = { + /* .references = */ { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8fdc9dce1..cfd5db3b4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2498,7 +2498,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - .references = rewriteRefs(), + /* .references = */ rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..39a31f0a0 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - .references = {}, + /* .references = */ {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - .references = {}, + /* .references = */ {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..9d2ea7156 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - .references = { + /* .references = */ { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..59a452918 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - .references = std::move(refs), + /* .references = */ std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..ff85b3932 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - .references = references, + /* .references = */ references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - .references = { + /* .references = */ { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..295ce4953 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -234,7 +234,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - .references = {}, + /* .references = */ {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -443,7 +443,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..28ddf2f4a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - .references = {}, + /* .references = */ {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..5de1aebfc 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - .references = {}, + /* .references = */ {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..bc270f66d 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - .references = {}, + /* .references = */ {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532..e552e8975 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -206,7 +206,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - .references = { + /* .references = */ { .others = std::move(references), .self = false, }, From 8c09888de4ada45ffdcb733879847aa24d3b00d8 Mon Sep 17 00:00:00 2001 From: Clemens Tolboom Date: Sat, 13 Feb 2021 15:18:08 +0100 Subject: [PATCH 026/114] Use long options instead of short ones It is a little hard to learn what the options mean. --- doc/manual/src/quick-start.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 651134c25..829e50d8a 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,7 +19,7 @@ to subsequent chapters. channel: ```console - $ nix-env -qaP + $ nix-env --query --available --attr-path nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 nixpkgs.firefox firefox-33.0.2 @@ -31,7 +31,7 @@ to subsequent chapters. 1. Install some packages from the channel: ```console - $ nix-env -iA nixpkgs.hello + $ nix-env --install --attr nixpkgs.hello ``` This should download pre-built packages; it should not build them @@ -49,13 +49,13 @@ to subsequent chapters. 1. Uninstall a package: ```console - $ nix-env -e hello + $ nix-env --uninstall hello ``` 1. You can also test a package without installing it: ```console - $ nix-shell -p hello + $ nix-shell --profile hello ``` This builds or downloads GNU Hello and its dependencies, then drops @@ -76,7 +76,7 @@ to subsequent chapters. ```console $ nix-channel --update nixpkgs - $ nix-env -u '*' + $ nix-env --upgrade '*' ``` The latter command will upgrade each installed package for which @@ -95,5 +95,5 @@ to subsequent chapters. them: ```console - $ nix-collect-garbage -d + $ nix-collect-garbage --delete-old ``` From ddb40ddd476164a316d5cee299d807856c251040 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 19 Feb 2023 20:00:02 -0500 Subject: [PATCH 027/114] Update doc/manual/src/quick-start.md Co-authored-by: Valentin Gagarin --- doc/manual/src/quick-start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 829e50d8a..1d2688ede 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -55,7 +55,7 @@ to subsequent chapters. 1. You can also test a package without installing it: ```console - $ nix-shell --profile hello + $ nix-shell --packages hello ``` This builds or downloads GNU Hello and its dependencies, then drops From c36b584f8eb103afa152ef4304cf9fd5c3ebaaf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:34:18 -0500 Subject: [PATCH 028/114] Fix typo in the method name --- src/libstore/path-info.cc | 4 ++-- src/libstore/path-info.hh | 2 +- src/libstore/store-api.cc | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ff85b3932..074b50818 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,7 +21,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional ValidPathInfo::contentAddressWithReferenences() const +std::optional ValidPathInfo::contentAddressWithReferences() const { if (! ca) return std::nullopt; @@ -54,7 +54,7 @@ std::optional ValidPathInfo::contentAddressWithRef bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = contentAddressWithReferences(); if (! fullCaOpt) return false; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 35aced472..97eb6638b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -78,7 +78,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional contentAddressWithReferenences() const; + std::optional contentAddressWithReferences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06d746a0b..73dcaf150 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -996,7 +996,7 @@ void copyStorePath( auto info2 = make_ref(*info); info2->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), - info->contentAddressWithReferenences().value()); + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1110,7 @@ std::map copyPaths( if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) From 123b11ff83da0cbcef6e4aae276bfbdd7a183656 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:49:13 -0500 Subject: [PATCH 029/114] Clarify store path grammar and improve comment on `makeType` --- src/libstore/store-api.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 73dcaf150..2ff92c0e6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). is the name of the output (usually, "out"). -

= base-16 representation of a SHA-256 hash of: +

= base-16 representation of a SHA-256 hash of + + = if = "text:...": the string written to the resulting store path - if = "source": + if = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if = "output:": @@ -164,8 +166,8 @@ StorePath Store::makeOutputPath(std::string_view id, /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ + hacky, but we can't put them in, say, (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, From 85bb865d200f04b73f183af722757c78d5a3be76 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 11:57:20 -0500 Subject: [PATCH 030/114] Revert "Remove some designated initializers" This reverts commit ee9eb83a842eb97d0180fd9d349d30ff27fdb485. --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 4 ++-- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 16 files changed, 21 insertions(+), 21 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index db733ce40..fca7607d3 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -300,7 +300,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 762ff8249..a54cca5ab 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1287,7 +1287,7 @@ drvName, Bindings * attrs, Value & v) .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", @@ -2103,7 +2103,7 @@ static void addPath( .method = method, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ffc6f5859..93592290f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -250,7 +250,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, - /* .references = */ {} + .references = {} }); if (state.store->isValidPath(expectedPath)) { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index dae4998f9..3936eadfe 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -215,7 +215,7 @@ StorePath Input::computeStorePath(Store & store) const .method = FileIngestionMethod::Recursive, .hash = *narHash, }, - /* .references = */ {}, + .references = {}, }); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b6f72bb1f..302046610 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -78,7 +78,7 @@ DownloadFileResult downloadFile( .method = FileIngestionMethod::Flat, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, hashString(htSHA256, sink.s), }; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5617e2c42..9cb0f74f6 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -313,7 +313,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n .method = method, .hash = nar.first, }, - /* .references = */ { + .references = { .others = references, .self = false, }, @@ -431,7 +431,7 @@ StorePath BinaryCacheStore::addToStore( .method = method, .hash = h, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 472188fea..765bb8f35 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2446,7 +2446,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() .method = outputHash.method, .hash = got, }, - /* .references = */ rewriteRefs(), + .references = rewriteRefs(), }, Hash::dummy, }; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 39a31f0a0..a51646d0f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -167,13 +167,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { h, - /* .references = */ {}, + .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { h, - /* .references = */ {}, + .references = {}, }; }, }, ca); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9d2ea7156..c6f870dde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1419,7 +1419,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name .method = method, .hash = hash, }, - /* .references = */ { + .references = { .others = references, .self = false, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 59a452918..42de79226 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -56,7 +56,7 @@ std::map makeContentAddressed( .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, - /* .references = */ std::move(refs), + .references = std::move(refs), }, Hash::dummy, }; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 074b50818..2a03e9dfa 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,7 +31,7 @@ std::optional ValidPathInfo::contentAddressWithRef assert(references.count(path) == 0); return TextInfo { th, - /* .references = */ references, + .references = references, }; }, [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { @@ -43,7 +43,7 @@ std::optional ValidPathInfo::contentAddressWithRef } return FixedOutputInfo { foh, - /* .references = */ { + .references = { .others = std::move(refs), .self = hasSelfReference, }, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2ff92c0e6..1c01c9cd8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -236,7 +236,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, .method = method, .hash = h, }, - /* .references = */ {}, + .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, .method = method, .hash = hash, }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 28ddf2f4a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,7 +220,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, - /* .references = */ {}, + .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5de1aebfc..81dbc09a6 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -49,7 +49,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand .method = std::move(ingestionMethod), .hash = std::move(hash), }, - /* .references = */ {}, + .references = {}, }, narHash, }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index bc270f66d..df9933d29 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -72,7 +72,7 @@ std::tuple prefetchFile( .method = ingestionMethod, .hash = *expectedHash, }, - /* .references = */ {}, + .references = {}, }); if (store->isValidPath(*storePath)) hash = expectedHash; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 28c7fe32d..5505d8bdd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -207,7 +207,7 @@ struct ProfileManifest .method = FileIngestionMethod::Recursive, .hash = narHash, }, - /* .references = */ { + .references = { .others = std::move(references), .self = false, }, From d381248ec0847cacd918480e83a99287f814456a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 28 Feb 2023 12:13:43 -0500 Subject: [PATCH 031/114] No inheritance for `TextInfo` and `FixedOutputInfo` --- perl/lib/Nix/Store.xs | 2 +- src/libexpr/primops.cc | 4 ++-- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/tarball.cc | 2 +- src/libstore/binary-cache-store.cc | 4 ++-- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 4 ++-- src/libstore/content-address.hh | 10 ++++++---- src/libstore/local-store.cc | 2 +- src/libstore/make-content-addressed.cc | 2 +- src/libstore/path-info.cc | 4 ++-- src/libstore/store-api.cc | 16 ++++++++-------- src/nix-store/nix-store.cc | 2 +- src/nix/add-to-store.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/profile.cc | 2 +- 17 files changed, 33 insertions(+), 31 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index fca7607d3..bfe00d3e2 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -296,7 +296,7 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a54cca5ab..7af796aa6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1283,7 +1283,7 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, @@ -2099,7 +2099,7 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = *expectedHash, }, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 93592290f..f3dce2214 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - { + .hash = { .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -211,7 +211,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = *narHash, }, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -74,7 +74,7 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Flat, .hash = hash, }, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9cb0f74f6..9eae8d534 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,7 +309,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = nar.first, }, @@ -427,7 +427,7 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 765bb8f35..87eac6e19 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2442,7 +2442,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store, outputPathName(drv->name, outputName), FixedOutputInfo { - { + .hash = { .method = outputHash.method, .hash = got, }, diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a51646d0f..d9a8a4535 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -166,13 +166,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { - h, + .hash = h, .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { - h, + .hash = h, .references = {}, }; }, diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c49ab269f..9fae288d8 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -111,18 +111,20 @@ struct StoreReferences { */ // This matches the additional info that we need for makeTextPath -struct TextInfo : TextHash { +struct TextInfo { + TextHash hash; // References for the paths, self references disallowed StorePathSet references; - GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); + GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo : FixedOutputHash { +struct FixedOutputInfo { + FixedOutputHash hash; // References for the paths StoreReferences references; - GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; typedef std::variant< diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6f870dde..8b33b2da5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1415,7 +1415,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 42de79226..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2a03e9dfa..76cab63e0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -30,7 +30,7 @@ std::optional ValidPathInfo::contentAddressWithRef [&](const TextHash & th) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - th, + .hash = th, .references = references, }; }, @@ -42,7 +42,7 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - foh, + .hash = foh, .references = { .others = std::move(refs), .self = hasSelfReference, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1c01c9cd8..b8a77b324 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash, + info.hash.hash, name); } @@ -232,7 +232,7 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - { + .hash = { .method = method, .hash = h, }, @@ -441,7 +441,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..735d6a592 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -216,7 +216,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,7 +45,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - { + .hash = { .method = std::move(ingestionMethod), .hash = std::move(hash), }, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..209517b21 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -68,7 +68,7 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - { + .hash = { .method = ingestionMethod, .hash = *expectedHash, }, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5505d8bdd..04ac48f00 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -203,7 +203,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narHash, }, From d1d1ae7a3b97059af09dd5a5dde2e37ada0fddac Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Mon, 27 Mar 2023 17:12:49 -0700 Subject: [PATCH 032/114] Documentation: list experimental features in manual Lists all current experimental features in the `nix.conf` manual. --- src/libutil/config.hh | 2 +- src/libutil/experimental-features.cc | 91 +++++++++++++++++++++++----- src/libutil/experimental-features.hh | 5 +- 3 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 748d6043b..59a766034 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -367,7 +367,7 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; + getExperimentalFeaturesList()}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 58d762ebb..fc8590674 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,18 +5,61 @@ namespace nix { -std::map stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, - { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::Cgroups, "cgroups" }, - { Xp::DiscardReferences, "discard-references" }, +std::map> stringifiedXpFeatures = { + { Xp::CaDerivations, {"ca-derivations", R"( + Allows derivations to be content-addressed in order to prevent rebuilds + when changes to the derivation do not result in changes to the + derivation's output. See + [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) + for more info. + )"} }, + { Xp::ImpureDerivations, {"impure-derivations", R"( + Allows derivations to produce non-fixed outputs by setting the `__impure` + derivation attribute to `true`. See [these release + notes](../release-notes/rl-2.8.md) for an example. + )"} }, + { Xp::Flakes, {"flakes", R"( + Allows for derivations to be packaged in flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed + introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + )"} }, + { Xp::NixCommand, {"nix-command", R"( + Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix + develop`, `nix run`, etc. See the manual for + [`nix`](../command-ref/new-cli/nix.md) for more info. + )"} }, + { Xp::RecursiveNix, {"recursive-nix", R"( + Allow Nix derivations to call Nix in order to recursively build derivations. + See [this + commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) + for more info. + )"} }, + { Xp::NoUrlLiterals, {"no-url-literals", R"( + Disallows unquoted URLs as part of the Nix language syntax. See [RFC + 45](https://github.com/NixOS/rfcs/pull/45) for more info. + )"} }, + { Xp::FetchClosure, {"fetch-closure", R"( + Enables the use of the `fetchClosure` function in the standard library. See + the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) + for more info. + )"} }, + { Xp::ReplFlake, {"repl-flake", R"( + Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` + or `nix repl .#foo`. + )"} }, + { Xp::AutoAllocateUids, {"auto-allocate-uids", R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + )"} }, + { Xp::Cgroups, {"cgroups", R"( + Allows Nix to execute builds inside cgroups. See + [`use-cgroups`](#conf-use-cgroups) for more info. + )"} }, + { Xp::DiscardReferences, {"discard-references", R"( + Enables the use of the `unsafeDiscardReferences` attribute in derivations + that use structured attributes. This disables scanning of outputs for + runtime dependencies. + )"} }, }; const std::optional parseExperimentalFeature(const std::string_view & name) @@ -26,8 +69,11 @@ const std::optional parseExperimentalFeature(const std::str static auto reverseXpMap = []() { auto reverseXpMap = std::make_unique(); - for (auto & [feature, name] : stringifiedXpFeatures) + std::string_view name; + for (auto & [feature, featureStringPair] : stringifiedXpFeatures) { + name = featureStringPair.first; (*reverseXpMap)[name] = feature; + } return reverseXpMap; }(); @@ -41,7 +87,24 @@ std::string_view showExperimentalFeature(const ExperimentalFeature feature) { const auto ret = get(stringifiedXpFeatures, feature); assert(ret); - return *ret; + return ret->first; +} + +std::string getExperimentalFeaturesList() { + std::string experimentalFeaturesList = R"( + Experimental Nix features to enable. + Current experimental features are the following: + +)"; + + std::string experimentalFeatureString; + for (auto& [feature, featureStringPair] : stringifiedXpFeatures) { + experimentalFeatureString = " - `" + featureStringPair.first + "`\n"; + experimentalFeatureString += featureStringPair.second + "\n\n"; + experimentalFeaturesList += experimentalFeatureString; + } + + return experimentalFeaturesList; } std::set parseFeatures(const std::set & rawFeatures) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index ac372e03e..1f1852705 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -10,8 +10,8 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. + * If you update this, don’t forget to also change the map defining their string + * representation and documentation in the corresponding `.cc` file as well. **/ enum struct ExperimentalFeature { @@ -36,6 +36,7 @@ using Xp = ExperimentalFeature; const std::optional parseExperimentalFeature( const std::string_view & name); std::string_view showExperimentalFeature(const ExperimentalFeature); +std::string getExperimentalFeaturesList(); std::ostream & operator<<( std::ostream & str, From a6d00a7bfb18e7ec461ac1d54203cc628aca5c66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 16:29:13 -0400 Subject: [PATCH 033/114] Fix warning --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9eae8d534..628e9b9db 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,7 +450,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, TextInfo { textHash, references }); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; From c51d554c933b5fe294da41fcdf5afe0d4f33f586 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 17:12:49 -0400 Subject: [PATCH 034/114] Use "raw pattern" for content address types We weren't because this ancient PR predated it! This is actually a new version of the pattern which addresses some issues identified in #7479. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 2 +- src/libstore/content-address.cc | 47 +++++---- src/libstore/content-address.hh | 110 +++++++++++++------- src/libstore/daemon.cc | 10 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-store.cc | 8 +- src/libstore/nar-info-disk-cache.cc | 2 +- src/libstore/nar-info.cc | 2 +- src/libstore/path-info.cc | 6 +- src/libstore/remote-store.cc | 8 +- src/libstore/store-api.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix/prefetch.cc | 2 +- 14 files changed, 118 insertions(+), 87 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index caa15ab04..4fb7aa9d8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2508,7 +2508,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Check wanted hash */ const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 87fed495c..190fb455a 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -97,7 +97,7 @@ void PathSubstitutionGoal::tryNext() if (ca) { subPath = sub->makeFixedOutputPathFromCA( std::string { storePath.name() }, - caWithoutRefs(*ca)); + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 64daea0d4..055b216db 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,6 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -22,35 +21,35 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) } } -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { + [](const TextHash & th) { return "text:" + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) { return "fixed:" + makeFileIngestionPrefix(fsh.method) + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextHashMethod & th) { + [](const TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod & fshm) { + [](const FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { @@ -94,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); @@ -112,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), }); }, - }, caMethod); + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -123,26 +122,28 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional parseContentAddressOpt(std::string_view rawCaOpt) +std::optional ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +const Hash & ContentAddress::getHash() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; }, - }, ca); + }, raw); } bool StoreReferences::empty() const @@ -155,7 +156,7 @@ size_t StoreReferences::size() const return (self ? 1 : 0) + others.size(); } -ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { @@ -169,7 +170,7 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { .references = {}, }; }, - }, ca); + }, ca.raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d74d1ff4b..d1dd1256c 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -39,17 +39,16 @@ enum struct FileIngestionMethod : uint8_t { Recursive = true }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; -}; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. */ std::string makeFileIngestionPrefix(FileIngestionMethod m); +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; /** * An enumeration of all the ways we can serialize file system objects. @@ -59,14 +58,25 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m); * with info on references, and we have `ContentAddressWithReferences`, * as defined further below. */ -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod -> ContentAddressMethod; +struct ContentAddressMethod +{ + typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > Raw; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward(arg)...) + { } + + static ContentAddressMethod parse(std::string_view rawCaMethod); + + std::string render() const; +}; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); /* * Mini content address @@ -115,25 +125,41 @@ struct FixedOutputHash { * - ‘fixed:::’: For paths computed by * Store::makeFixedOutputPath() / Store::addToStore(). */ -typedef std::variant< - TextHash, - FixedOutputHash -> ContentAddress; +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; -/** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). - */ -std::string renderContentAddress(ContentAddress ca); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional parseOpt(std::string_view rawCaOpt); + + const Hash & getHash() const; +}; std::string renderContentAddress(std::optional ca); -ContentAddress parseContentAddress(std::string_view rawCa); - -std::optional parseContentAddressOpt(std::string_view rawCaOpt); - -Hash getContentAddressHash(const ContentAddress & ca); +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ /** * A set of references to other store objects. @@ -167,12 +193,6 @@ struct StoreReferences { GENERATE_CMP(StoreReferences, me->self, me->others); }; -/* - * Full content address - * - * See the schema for store paths in store-api.cc - */ - // This matches the additional info that we need for makeTextPath struct TextInfo { TextHash hash; @@ -200,15 +220,25 @@ struct FixedOutputInfo { * * A ContentAddress without a Hash. */ -typedef std::variant< - TextInfo, - FixedOutputInfo -> ContentAddressWithReferences; +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; -/** - * Create a ContentAddressWithReferences from a mere ContentAddress, by - * assuming no references in all cases. - */ -ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + Raw raw; + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward(arg)...) + { } + + /** + * Create a ContentAddressWithReferences from a mere ContentAddress, by + * assuming no references in all cases. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &); +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 656ad4587..0169eef1a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { + [&](const FixedOutputHashMethod & fohm) { auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref store, info.references = worker_proto::read(*store, from, Phantom {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 98322b045..a1c38d180 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings(conn->from); auto s = readString(conn->from); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e1c7e387a..b49d5462b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -944,7 +944,7 @@ std::shared_ptr LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1150,7 +1150,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst if (path.second) { subPath = makeFixedOutputPathFromCA( path.first.name(), - caWithoutRefs(*path.second)); + ContentAddressWithReferences::withoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1329,7 +1329,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&*info.ca)) { + if (auto foHash = std::get_if(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1342,7 +1342,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if(&*info.ca)) { + if (auto textHash = std::get_if(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 2645f468b..c7176d30f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -273,7 +273,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..274cd861c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 76cab63e0..e60d7abe0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -49,7 +49,7 @@ std::optional ValidPathInfo::contentAddressWithRef }, }; }, - }, *ca); + }, ca->raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -116,7 +116,7 @@ ValidPathInfo::ValidPathInfo( this->references.insert(path); this->ca = std::move((FixedOutputHash &&) foi); }, - }, std::move(ca)); + }, std::move(ca).raw); } @@ -136,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned if (format >= 16) { source >> info.ultimate; info.sigs = readStrings(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d24d83117..ac98e76d2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -44,7 +44,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath) ContentAddress read(const Store & store, Source & from, Phantom _) { - return parseContentAddress(readString(from)); + return ContentAddress::parse(readString(from)); } void write(const Store & store, Sink & out, const ContentAddress & ca) @@ -134,7 +134,7 @@ void write(const Store & store, Sink & out, const std::optional & sto std::optional read(const Store & store, Source & from, Phantom> _) { - return parseContentAddressOpt(readString(from)); + return ContentAddress::parseOpt(readString(from)); } void write(const Store & store, Sink & out, const std::optional & caOpt) @@ -545,7 +545,7 @@ ref RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << caMethod.render(); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -603,7 +603,7 @@ ref RemoteStore::addCAToStore( } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fed38e2dd..78b0d907e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -221,7 +221,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA [&](const FixedOutputInfo & foi) { return makeFixedOutputPath(name, foi); } - }, ca); + }, ca.raw); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 26febb6e3..3d2dc49fd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = worker_proto::read(*store, in, Phantom {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b06b8a320..56e7bbb6e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -124,7 +124,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; From 5d56e2daf70788fae532d1875edbd4e9bdb5afef Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Apr 2023 16:52:23 -0400 Subject: [PATCH 035/114] Add comparison methods for content addresses --- src/libstore/content-address.hh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index d1dd1256c..737bf9a41 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -46,8 +46,10 @@ enum struct FileIngestionMethod : uint8_t { std::string makeFileIngestionPrefix(FileIngestionMethod m); struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + FileIngestionMethod fileIngestionMethod; + HashType hashType; + + GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); }; /** @@ -67,6 +69,8 @@ struct ContentAddressMethod Raw raw; + GENERATE_CMP(ContentAddressMethod, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressMethod(auto &&... arg) : raw(std::forward(arg)...) @@ -134,6 +138,8 @@ struct ContentAddress Raw raw; + GENERATE_CMP(ContentAddress, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddress(auto &&... arg) : raw(std::forward(arg)...) @@ -229,6 +235,8 @@ struct ContentAddressWithReferences Raw raw; + GENERATE_CMP(ContentAddressWithReferences, me->raw); + /* The moral equivalent of `using Raw::Raw;` */ ContentAddressWithReferences(auto &&... arg) : raw(std::forward(arg)...) From 2585bcaa500f874871d8ac34a83fca638770b78f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 18:11:16 -0400 Subject: [PATCH 036/114] Rework a few things with the experimental features list - Use struct not `std::pair`, designated initializers - Use `constexpr` array that we can index by enum tag - It no longer segfaults; not sure why. --- src/libutil/experimental-features.cc | 209 +++++++++++++++++---------- src/libutil/experimental-features.hh | 5 +- src/libutil/util.hh | 10 +- 3 files changed, 142 insertions(+), 82 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index fc8590674..305b0cb11 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,75 +5,131 @@ namespace nix { -std::map> stringifiedXpFeatures = { - { Xp::CaDerivations, {"ca-derivations", R"( - Allows derivations to be content-addressed in order to prevent rebuilds - when changes to the derivation do not result in changes to the - derivation's output. See - [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) - for more info. - )"} }, - { Xp::ImpureDerivations, {"impure-derivations", R"( - Allows derivations to produce non-fixed outputs by setting the `__impure` - derivation attribute to `true`. See [these release - notes](../release-notes/rl-2.8.md) for an example. - )"} }, - { Xp::Flakes, {"flakes", R"( - Allows for derivations to be packaged in flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed - introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. - )"} }, - { Xp::NixCommand, {"nix-command", R"( - Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix - develop`, `nix run`, etc. See the manual for - [`nix`](../command-ref/new-cli/nix.md) for more info. - )"} }, - { Xp::RecursiveNix, {"recursive-nix", R"( - Allow Nix derivations to call Nix in order to recursively build derivations. - See [this - commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) - for more info. - )"} }, - { Xp::NoUrlLiterals, {"no-url-literals", R"( - Disallows unquoted URLs as part of the Nix language syntax. See [RFC - 45](https://github.com/NixOS/rfcs/pull/45) for more info. - )"} }, - { Xp::FetchClosure, {"fetch-closure", R"( - Enables the use of the `fetchClosure` function in the standard library. See - the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) - for more info. - )"} }, - { Xp::ReplFlake, {"repl-flake", R"( - Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` - or `nix repl .#foo`. - )"} }, - { Xp::AutoAllocateUids, {"auto-allocate-uids", R"( - Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. - )"} }, - { Xp::Cgroups, {"cgroups", R"( - Allows Nix to execute builds inside cgroups. See - [`use-cgroups`](#conf-use-cgroups) for more info. - )"} }, - { Xp::DiscardReferences, {"discard-references", R"( - Enables the use of the `unsafeDiscardReferences` attribute in derivations - that use structured attributes. This disables scanning of outputs for - runtime dependencies. - )"} }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; }; +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allows derivations to be content-addressed in order to prevent rebuilds + when changes to the derivation do not result in changes to the + derivation's output. See + [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) + for more info. + )", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allows derivations to produce non-fixed outputs by setting the `__impure` + derivation attribute to `true`. See [these release + notes](../release-notes/rl-2.8.md) for an example. + )", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Allows for derivations to be packaged in flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed + introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + )", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix + develop`, `nix run`, etc. See the manual for + [`nix`](../command-ref/new-cli/nix.md) for more info. + )", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow Nix derivations to call Nix in order to recursively build derivations. + See [this + commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) + for more info. + )", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallows unquoted URLs as part of the Nix language syntax. See [RFC + 45](https://github.com/NixOS/rfcs/pull/45) for more info. + )", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enables the use of the `fetchClosure` function in the standard library. See + the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) + for more info. + )", + }, + { + .tag = Xp::ReplFlake, + .name = "repl-flake", + .description = R"( + Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` + or `nix repl .#foo`. + )", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + )", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + [`use-cgroups`](#conf-use-cgroups) for more info. + )", + }, + { + .tag = Xp::DiscardReferences, + .name = "discard-references", + .description = R"( + Enables the use of the `unsafeDiscardReferences` attribute in derivations + that use structured attributes. This disables scanning of outputs for + runtime dependencies. + )", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; - static auto reverseXpMap = []() - { + static std::unique_ptr reverseXpMap = [](){ auto reverseXpMap = std::make_unique(); - std::string_view name; - for (auto & [feature, featureStringPair] : stringifiedXpFeatures) { - name = featureStringPair.first; - (*reverseXpMap)[name] = feature; - } + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -83,25 +139,29 @@ const std::optional parseExperimentalFeature(const std::str return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) { - const auto ret = get(stringifiedXpFeatures, feature); - assert(ret); - return ret->first; + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; } std::string getExperimentalFeaturesList() { std::string experimentalFeaturesList = R"( - Experimental Nix features to enable. - Current experimental features are the following: + Experimental Nix features to enable. + Current experimental features are the following: )"; - std::string experimentalFeatureString; - for (auto& [feature, featureStringPair] : stringifiedXpFeatures) { - experimentalFeatureString = " - `" + featureStringPair.first + "`\n"; - experimentalFeatureString += featureStringPair.second + "\n\n"; - experimentalFeaturesList += experimentalFeatureString; + for (auto & xpFeature : xpFeatureDetails) { + experimentalFeaturesList += std::string {} + /* length of this first string must be 12, matching the indent of + the descriptions in the xpFeatureDetails literal. FIXME compute + markdown in a less hacky way. */ + + " - " + + "`" + xpFeature.name + "`" + + "\n" + + xpFeature.description + + "\n\n"; } return experimentalFeaturesList; @@ -110,10 +170,9 @@ std::string getExperimentalFeaturesList() { std::set parseFeatures(const std::set & rawFeatures) { std::set res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 7c2f872c5..650fa6533 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -11,8 +11,9 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their string - * representation and documentation in the corresponding `.cc` file as well. + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. */ enum struct ExperimentalFeature { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6c2706cc1..94c9efb74 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -730,16 +730,16 @@ constexpr auto enumerate(T && iterable) { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward(iterable) }; From 32d72b1696c085d17c82c9f9b02f31ac32aa148f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 18:57:46 -0400 Subject: [PATCH 037/114] Add more API docs to `experimental-features.hh` --- src/libutil/experimental-features.hh | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 650fa6533..3c479dbd9 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -35,27 +35,54 @@ enum struct ExperimentalFeature */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); + +/** + * Compute the documentation of all experimental features. + * + * This a markdown bulleted list where each item is first (a) the + * experimental feature flag name in backticks, and then (b) the + * description of the experimental feature. + */ std::string getExperimentalFeaturesList(); +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set parseFeatures(const std::set &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; /** From 7963d0c68dfd31a1aa179a8bd8f37dd790701d7c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 23:11:21 -0400 Subject: [PATCH 038/114] Try auto-labeling a few more things --- .github/labeler.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index dc502b6d5..fce0d3aeb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,5 +2,22 @@ - doc/manual/* - src/nix/**/*.md +"store": + - src/libstore/store-api.* + - src/libstore/*-store.* + +"fetching": + - src/libfetchers/**/* + +"repl": + - src/libcmd/repl.* + - src/nix/repl.* + +"new-cli": + - src/nix/**/* + "tests": + # Unit tests + - src/*/tests/**/* + # Functional and integration tests - tests/**/* From 7076d37047d73e82cb51c315fede4906a76f57b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 3 Apr 2023 20:47:21 +0200 Subject: [PATCH 039/114] Fix the flaky `nix-profile` test Exclude the `error (ignored)` from the message that is checked by the install conflict test. Fix https://github.com/NixOS/nix/issues/8140 --- tests/nix-profile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 652e8a8f2..4ef5b484a 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -144,6 +144,7 @@ expect 1 nix profile install $flake2Dir diff -u <( nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ || true ) <(cat << EOF error: An existing package already provides the following file: From bdeeffff967a513b6a35165bb476d24491a03e23 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Apr 2023 19:16:10 -0400 Subject: [PATCH 040/114] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 41 ++++++++++++++-------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 305b0cb11..a64e9715a 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -17,11 +17,11 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::CaDerivations, .name = "ca-derivations", .description = R"( - Allows derivations to be content-addressed in order to prevent rebuilds + Allow derivations to be content-addressed in order to prevent rebuilds when changes to the derivation do not result in changes to the derivation's output. See - [__contentAddressed](../language/advanced-attributes.md#adv-attr-__contentAddressed) - for more info. + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. )", }, { @@ -37,18 +37,16 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::Flakes, .name = "flakes", .description = R"( - Allows for derivations to be packaged in flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) or this [detailed - introduction](https://www.tweag.io/blog/2020-05-25-flakes/) for more info. + Enable flakes. See the manual entry for + [`nix flake`](../command-ref/new-cli/nix3-flake.md) for details. )", }, { .tag = Xp::NixCommand, .name = "nix-command", .description = R"( - Allows the usage of the new `nix` CLI subcommands, such as `nix build`, `nix - develop`, `nix run`, etc. See the manual for - [`nix`](../command-ref/new-cli/nix.md) for more info. + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, { @@ -73,17 +71,14 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::FetchClosure, .name = "fetch-closure", .description = R"( - Enables the use of the `fetchClosure` function in the standard library. See - the docs for [`fetchClosure`](../language/builtins.md#builtins-fetchClosure) - for more info. + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. )", }, { .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( - Allows the user to enter a Nix REPL within a flake, e.g. `nix repl nixpkgs` - or `nix repl .#foo`. + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, { @@ -91,7 +86,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See [here](#conf-auto-allocate-uids) for more info. + `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. )", }, { @@ -99,15 +94,15 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - [`use-cgroups`](#conf-use-cgroups) for more info. + the [`use-cgroups`](#conf-use-cgroups) setting for details. )", }, { .tag = Xp::DiscardReferences, .name = "discard-references", .description = R"( - Enables the use of the `unsafeDiscardReferences` attribute in derivations - that use structured attributes. This disables scanning of outputs for + Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations + that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for runtime dependencies. )", }, @@ -147,8 +142,14 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) std::string getExperimentalFeaturesList() { std::string experimentalFeaturesList = R"( - Experimental Nix features to enable. - Current experimental features are the following: + Experimental features that can be enabled. + + Example: + + ``` + experimental-features = nix-command flakes + + Experimental features available: )"; From 53d0836347ea262884658036f97bf19ecbdc5c26 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Apr 2023 22:57:11 -0400 Subject: [PATCH 041/114] Assemble experimental feature docs outside of Nix itself Instead of constructing a markdown list in C++ (which involved all sorts of nasty string literals), export some JSON and assemble it with the manual build system. Besides following the precedent set with other dumped data, this is a better separate of content and presentation; if we decide for example we want to display this information in a different way, or in a different section of the manual, it will become much easier to do so. --- .gitignore | 2 ++ doc/manual/generate-xp-features.nix | 11 ++++++++++ doc/manual/local.mk | 13 +++++++++-- doc/manual/utils.nix | 6 +++--- src/libutil/config.hh | 17 +++++++++++++-- src/libutil/experimental-features.cc | 32 ++++++---------------------- src/libutil/experimental-features.hh | 6 ++---- src/nix/main.cc | 5 +++++ 8 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 doc/manual/generate-xp-features.nix diff --git a/.gitignore b/.gitignore index e326966d6..53442751f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,9 +19,11 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md +/doc/manual/src/command-ref/experimental-features.md /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix new file mode 100644 index 000000000..db1ba6092 --- /dev/null +++ b/doc/manual/generate-xp-features.nix @@ -0,0 +1,11 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + squash '' + - [`${name}`](#xp-feature-${name}) + + ${indent " " doc} + ''; +in xps: indent " " (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index df941d460..0d4b9d640 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -88,12 +88,12 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,6 +106,15 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ +$(d)/src/command-ref/experimental-features.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 5eacce0dd..f78e6bb02 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -74,10 +74,10 @@ rec { if aliases == [] then "" else "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index a001056f7..8b0fe6555 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -371,8 +371,21 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { - Setting> experimentalFeatures{this, {}, "experimental-features", - getExperimentalFeaturesList()}; + Setting> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + Experimental features available: + + {{#include experimental-features.md}} + )"}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index a64e9715a..010ab1d68 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -140,32 +140,12 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -std::string getExperimentalFeaturesList() { - std::string experimentalFeaturesList = R"( - Experimental features that can be enabled. - - Example: - - ``` - experimental-features = nix-command flakes - - Experimental features available: - -)"; - - for (auto & xpFeature : xpFeatureDetails) { - experimentalFeaturesList += std::string {} - /* length of this first string must be 12, matching the indent of - the descriptions in the xpFeatureDetails literal. FIXME compute - markdown in a less hacky way. */ - + " - " - + "`" + xpFeature.name + "`" - + "\n" - + xpFeature.description - + "\n\n"; - } - - return experimentalFeaturesList; +nlohmann::json documentExperimentalFeatures() { + StringMap res; + for (auto & xpFeature : xpFeatureDetails) + res[std::string { xpFeature.name }] = + trim(stripIndentation(xpFeature.description)); + return (nlohmann::json) res; } std::set parseFeatures(const std::set & rawFeatures) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 3c479dbd9..8ef66263a 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -51,11 +51,9 @@ std::string_view showExperimentalFeature(const ExperimentalFeature); /** * Compute the documentation of all experimental features. * - * This a markdown bulleted list where each item is first (a) the - * experimental feature flag name in backticks, and then (b) the - * description of the experimental feature. + * See `doc/manual` for how this information is used. */ -std::string getExperimentalFeaturesList(); +nlohmann::json documentExperimentalFeatures(); /** * Shorthand for `str << showExperimentalFeature(feature)`. diff --git a/src/nix/main.cc b/src/nix/main.cc index 4d4164333..62b1f98d1 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -374,6 +374,11 @@ void mainWrapped(int argc, char * * argv) return; } + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); + return; + } + Finally printCompletions([&]() { if (completions) { From b11ae93581065d7fa98c69d26601ef1ecf211c0f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 26 Dec 2022 18:27:12 +0100 Subject: [PATCH 042/114] remove incorrect reference the semantics are not explained in the referenced section any more, they have been moved to the documentation for common options in the new CLI [0]. [0]: 703d863a48f549b2626382eda407ffae779f8725 --- doc/manual/src/command-ref/opt-common.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a23b87e4e..c94b6aef8 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,10 +203,9 @@ Most Nix commands accept the following command-line options: instead. - [`-I`](#opt-I) *path*\ - Add a path to the Nix expression search path. This option may be - given multiple times. See the `NIX_PATH` environment variable for - information on the semantics of the Nix search path. Paths added - through `-I` take precedence over `NIX_PATH`. + Add a path to the Nix expression search path. + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides From faefaac87568fe3ff04f4c2624f293763de2ae1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 5 Apr 2023 17:20:04 +0200 Subject: [PATCH 043/114] Explicitely define `LockFile::operator!=` It should be syntethised in terms of `operator==`, but the GCC version used on aarch64-linux doesn't implement that (see https://hydra.nixos.org/build/214848896=. So explicitely define it. Fix https://github.com/NixOS/nix/issues/8159 --- src/libexpr/flake/lockfile.cc | 5 +++++ src/libexpr/flake/lockfile.hh | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74e68c9c..ba2fd46f0 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON() == other.toJSON(); } +bool LockFile::operator !=(const LockFile & other) const +{ + return !(*this == other); +} + InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 6512509c5..4616a77f6 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -67,6 +67,9 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; + // Needed for old gcc versions that don't syntethise it (like gcc 8.2.2 + // that is still the default on aarch64-linux) + bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); From 94812cca98fbb157e5f64a15a85a2b852d289feb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Apr 2023 13:15:50 +0200 Subject: [PATCH 044/114] Backport SourcePath from the lazy-trees branch This introduces the SourcePath type from lazy-trees as an abstraction for accessing files from inputs that may not be materialized in the real filesystem (e.g. Git repositories). Currently, however, it's just a wrapper around CanonPath, so it shouldn't change any behaviour. (On lazy-trees, SourcePath is a tuple.) --- src/libcmd/common-eval-args.cc | 10 +- src/libcmd/common-eval-args.hh | 3 +- src/libcmd/editor-for.cc | 7 +- src/libcmd/editor-for.hh | 3 +- src/libcmd/installable-flake.cc | 3 +- src/libcmd/installables.cc | 2 +- src/libcmd/repl.cc | 12 +- src/libexpr/attr-path.cc | 26 +++-- src/libexpr/attr-path.hh | 2 +- src/libexpr/eval-cache.cc | 10 +- src/libexpr/eval.cc | 94 ++++++++-------- src/libexpr/eval.hh | 56 +++++----- src/libexpr/flake/flake.cc | 4 +- src/libexpr/parser.y | 52 ++++----- src/libexpr/paths.cc | 10 ++ src/libexpr/primops.cc | 122 ++++++++++---------- src/libexpr/tests/libexpr.hh | 2 +- src/libexpr/tests/local.mk | 2 +- src/libexpr/value-to-json.cc | 5 +- src/libexpr/value-to-xml.cc | 2 +- src/libexpr/value.hh | 28 ++++- src/libfetchers/input-accessor.cc | 100 +++++++++++++++++ src/libfetchers/input-accessor.hh | 149 +++++++++++++++++++++++++ src/libutil/canon-path.cc | 5 + src/libutil/canon-path.hh | 2 + src/nix-build/nix-build.cc | 6 +- src/nix-env/nix-env.cc | 70 ++++++------ src/nix-env/user-env.cc | 6 +- src/nix-instantiate/nix-instantiate.cc | 10 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 4 +- src/nix/main.cc | 6 +- src/nix/prefetch.cc | 11 +- src/nix/upgrade-nix.cc | 2 +- tests/plugins/local.mk | 2 +- 35 files changed, 567 insertions(+), 263 deletions(-) create mode 100644 src/libexpr/paths.cc create mode 100644 src/libfetchers/input-accessor.cc create mode 100644 src/libfetchers/input-accessor.hh diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 5b6477c82..ff3abd534 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -153,7 +153,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) for (auto & i : autoArgs) { auto v = state.allocValue(); if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath("."))); + state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd()))); else v->mkString(((std::string_view) i.second).substr(1)); res.insert(state.symbols.create(i.first), v); @@ -161,19 +161,19 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -Path lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; - return state.store->toRealPath(storePath); + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; - return state.store->toRealPath(storePath); + return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { @@ -182,7 +182,7 @@ Path lookupFileArg(EvalState & state, std::string_view s) } else - return absPath(std::string(s)); + return state.rootPath(CanonPath::fromCwd(s)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b69db11dd..83edcfb85 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -8,6 +8,7 @@ namespace nix { class Store; class EvalState; class Bindings; +struct SourcePath; struct MixEvalArgs : virtual Args { @@ -25,6 +26,6 @@ private: std::map autoArgs; }; -Path lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index f674f32bd..a17c6f12a 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -3,8 +3,11 @@ namespace nix { -Strings editorFor(const Path & file, uint32_t line) +Strings editorFor(const SourcePath & file, uint32_t line) { + auto path = file.getPhysicalPath(); + if (!path) + throw Error("cannot open '%s' in an editor because it has no physical path", file); auto editor = getEnv("EDITOR").value_or("cat"); auto args = tokenizeString(editor); if (line > 0 && ( @@ -13,7 +16,7 @@ Strings editorFor(const Path & file, uint32_t line) editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) args.push_back(fmt("+%d", line)); - args.push_back(file); + args.push_back(path->abs()); return args; } diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index f752bd849..c4873921f 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -2,11 +2,12 @@ ///@file #include "types.hh" +#include "input-accessor.hh" namespace nix { /* Helper function to generate args that invoke $EDITOR on filename:lineno. */ -Strings editorFor(const Path & file, uint32_t line); +Strings editorFor(const SourcePath & file, uint32_t line); } diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index a3352af76..19e982df1 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -96,8 +96,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto v = attr->forceValue(); if (v.type() == nPath) { - PathSet context; - auto storePath = state->copyPathToStore(context, Path(v.path)); + auto storePath = v.path().fetchToStore(state->store); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 67549b280..1873d175a 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -427,7 +427,7 @@ Installables SourceExprCommand::parseInstallables( else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { - auto e = state->parseExprFromString(*expr, absPath(".")); + auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); state->eval(e, *vFile); } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..9002fa555 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -54,8 +54,6 @@ struct NixRepl , gc #endif { - std::string curDir; - size_t debugTraceIndex; Strings loadedFiles; @@ -113,7 +111,6 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref store, refstaticBaseEnv.get())) , historyFile(getDataDir() + "/nix/repl-history") { - curDir = absPath("."); } @@ -590,7 +587,7 @@ bool NixRepl::processLine(std::string line) Value v; evalString(arg, v); - const auto [path, line] = [&] () -> std::pair { + const auto [path, line] = [&] () -> std::pair { if (v.type() == nPath || v.type() == nString) { PathSet context; auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); @@ -598,7 +595,7 @@ bool NixRepl::processLine(std::string line) } else if (v.isLambda()) { auto pos = state->positions[v.lambda.fun->pos]; if (auto path = std::get_if(&pos.origin)) - return {*path, pos.line}; + return {SourcePath(CanonPath(*path)), pos.line}; else throw Error("'%s' cannot be shown in an editor", pos); } else { @@ -872,8 +869,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v) Expr * NixRepl::parseString(std::string s) { - Expr * e = state->parseExprFromString(std::move(s), curDir, staticEnv); - return e; + return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv); } @@ -930,7 +926,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nPath: - str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? + str << ANSI_GREEN << v.path().to_string() << ANSI_NORMAL; // !!! escaping? break; case nNull: diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 7c0705091..8ae4270e6 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -106,7 +106,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin } -std::pair findPackageFilename(EvalState & state, Value & v, std::string what) +std::pair findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -118,21 +118,25 @@ std::pair findPackageFilename(EvalState & state, Value & // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - auto pos = state.forceString(*v2, noPos, "while evaluating the 'meta.position' attribute of a derivation"); + PathSet context; + auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation"); - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw ParseError("cannot parse meta.position attribute '%s'", pos); + auto fn = path.path.abs(); + + auto fail = [fn]() { + throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); + }; - std::string filename(pos, 0, colon); - unsigned int lineno; try { - lineno = std::stoi(std::string(pos, colon + 1, std::string::npos)); + auto colon = fn.rfind(':'); + if (colon == std::string::npos) fail(); + std::string filename(fn, 0, colon); + auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); + return {CanonPath(fn.substr(0, colon)), lineno}; } catch (std::invalid_argument & e) { - throw ParseError("cannot parse line number '%s'", pos); + fail(); + abort(); } - - return { std::move(filename), lineno }; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index d0d05b1a1..dee811fe1 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -18,7 +18,7 @@ std::pair findAlongAttrPath( Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ -std::pair findPackageFilename(EvalState & state, Value & v, std::string what); +std::pair findPackageFilename(EvalState & state, Value & v, std::string what); std::vector parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1219b2471..ec4ad2f5e 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -442,8 +442,10 @@ Value & AttrCursor::forceValue() if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type() == nPath) - cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + else if (v.type() == nPath) { + auto path = v.path().path; + cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; + } else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nInt) @@ -580,7 +582,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -622,7 +624,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) return {v.string.s, v.getContext(*root->state.store)}; else if (v.type() == nPath) - return {v.path, {}}; + return {v.path().to_string(), {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 584bbc879..3b2877602 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -118,7 +118,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "\""; break; case tPath: - str << path; // !!! escaping? + str << path().to_string(); // !!! escaping? break; case tNull: str << "null"; @@ -577,11 +577,11 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v.mkString(path, PathSet({path})); } -Path EvalState::checkSourcePath(const Path & path_) +SourcePath EvalState::checkSourcePath(const SourcePath & path_) { if (!allowedPaths) return path_; - auto i = resolvedPaths.find(path_); + auto i = resolvedPaths.find(path_.path.abs()); if (i != resolvedPaths.end()) return i->second; @@ -591,9 +591,9 @@ Path EvalState::checkSourcePath(const Path & path_) * attacker can't append ../../... to a path that would be in allowedPaths * and thus leak symlink targets. */ - Path abspath = canonPath(path_); + Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -611,11 +611,11 @@ Path EvalState::checkSourcePath(const Path & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - Path path = canonPath(abspath, true); + SourcePath path = CanonPath(canonPath(abspath, true)); for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) { - resolvedPaths[path_] = path; + if (isDirOrInDir(path.path.abs(), i)) { + resolvedPaths.insert_or_assign(path_.path.abs(), path); return path; } } @@ -643,12 +643,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(uri); + checkSourcePath(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(std::string(uri, 7)); + checkSourcePath(CanonPath(std::string(uri, 7))); return; } @@ -933,9 +933,9 @@ void Value::mkStringMove(const char * s, const PathSet & context) } -void Value::mkPath(std::string_view s) +void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(s)); + mkPath(makeImmutableString(path.path.abs())); } @@ -1049,7 +1049,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -1059,7 +1059,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) return; } - Path resolvedPath = resolveExprPath(path); + auto resolvedPath = resolveExprPath(path); if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; @@ -1087,8 +1087,8 @@ void EvalState::resetFileCache() void EvalState::cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial) @@ -1102,7 +1102,7 @@ void EvalState::cacheFile( *e, this->baseEnv, e->getPos() ? static_cast>(positions[e->getPos()]) : nullptr, - "while evaluating the file '%1%':", resolvedPath) + "while evaluating the file '%1%':", resolvedPath.to_string()) : nullptr; // Enforce that 'flake.nix' is a direct attrset, not a @@ -1112,7 +1112,7 @@ void EvalState::cacheFile( error("file '%s' must be an attribute set", path).debugThrow(); eval(e, v); } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); throw; } @@ -1947,7 +1947,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(canonPath(str())); + v.mkPath(CanonPath(canonPath(str()))); } else v.mkStringMove(c_str(), context); } @@ -2136,8 +2136,14 @@ std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context, - std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath) +BackedStringView EvalState::coerceToString( + const PosIdx pos, + Value & v, + PathSet & context, + std::string_view errorCtx, + bool coerceMore, + bool copyToStore, + bool canonicalizePath) { forceValue(v, pos); @@ -2147,12 +2153,14 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } if (v.type() == nPath) { - BackedStringView path(PathView(v.path)); - if (canonicalizePath) - path = canonPath(*path); - if (copyToStore) - path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); - return path; + return + !canonicalizePath && !copyToStore + ? // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + v._path + : copyToStore + ? store->printStorePath(copyPathToStore(context, v.path())) + : std::string(v.path().path.abs()); } if (v.type() == nAttrs) { @@ -2213,36 +2221,34 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } -StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(PathSet & context, const SourcePath & path) { - if (nix::isDerivation(path)) + if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto dstPath = [&]() -> StorePath - { - auto i = srcToStore.find(path); - if (i != srcToStore.end()) return i->second; + auto i = srcToStore.find(path); - auto dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - allowPath(dstPath); - srcToStore.insert_or_assign(path, dstPath); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); - return dstPath; - }(); + auto dstPath = i != srcToStore.end() + ? i->second + : [&]() { + auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); context.insert(store->printStorePath(dstPath)); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return path; + return CanonPath(path); } @@ -2285,7 +2291,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return strcmp(v1.string.s, v2.string.s) == 0; case nPath: - return strcmp(v1.path, v2.path) == 0; + return strcmp(v1._path, v2._path) == 0; case nNull: return true; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a1b54951e..1e8f8391c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -8,6 +8,7 @@ #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "input-accessor.hh" #include #include @@ -56,15 +57,11 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, PathSet & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ -typedef std::map SrcToStore; - - std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); +// FIXME: maybe change this to an std::variant. typedef std::pair SearchPathElem; typedef std::list SearchPath; @@ -217,21 +214,24 @@ public: } private: - SrcToStore srcToStore; + + /* Cache for calls to addToStore(); maps source paths to the store + paths. */ + std::map srcToStore; /* A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; + typedef std::map, traceable_allocator>> FileParseCache; #else - typedef std::map FileParseCache; + typedef std::map FileParseCache; #endif FileParseCache fileParseCache; /* A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; + typedef std::map, traceable_allocator>> FileEvalCache; #else - typedef std::map FileEvalCache; + typedef std::map FileEvalCache; #endif FileEvalCache fileEvalCache; @@ -240,7 +240,7 @@ private: std::map> searchPathResolved; /* Cache used by checkSourcePath(). */ - std::unordered_map resolvedPaths; + std::unordered_map resolvedPaths; /* Cache used by prim_match(). */ std::shared_ptr regexCache; @@ -265,6 +265,12 @@ public: SearchPath getSearchPath() { return searchPath; } + /** + * Return a `SourcePath` that refers to `path` in the root + * filesystem. + */ + SourcePath rootPath(CanonPath path); + /* Allow access to a path. */ void allowPath(const Path & path); @@ -277,7 +283,7 @@ public: /* Check whether access to a path is allowed and throw an error if not. Otherwise return the canonicalised path. */ - Path checkSourcePath(const Path & path); + SourcePath checkSourcePath(const SourcePath & path); void checkURI(const std::string & uri); @@ -291,24 +297,24 @@ public: Path toRealPath(const Path & path, const PathSet & context); /* Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - Expr * parseExprFromFile(const Path & path, std::shared_ptr & staticEnv); + Expr * parseExprFromFile(const SourcePath & path); + Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv); /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr & staticEnv); - Expr * parseExprFromString(std::string s, const Path & basePath); + Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr & staticEnv); + Expr * parseExprFromString(std::string s, const SourcePath & basePath); Expr * parseStdin(); /* Evaluate an expression read from the given file to normal form. Optionally enforce that the top-level expression is trivial (i.e. doesn't require arbitrary computation). */ - void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); + void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); /* Like `evalFile`, but with an already parsed expression. */ void cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial = false); @@ -316,8 +322,8 @@ public: void resetFileCache(); /* Look up a file in the search path. */ - Path findFile(const std::string_view path); - Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const std::string_view path); + SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /* If the specified search path element is a URI, download it. */ std::pair resolveSearchPathElem(const SearchPathElem & elem); @@ -383,12 +389,12 @@ public: bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(PathSet & context, const SourcePath & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + SourcePath coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); /* Like coerceToPath, but the result must be a store path. */ StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); @@ -444,7 +450,7 @@ private: char * text, size_t length, Pos::Origin origin, - Path basePath, + const SourcePath & basePath, std::shared_ptr & staticEnv); public: @@ -556,7 +562,7 @@ std::string_view showType(ValueType type); std::string showType(const Value & v); /* If `path' refers to a directory, then append "/default.nix". */ -Path resolveExprPath(Path path); +SourcePath resolveExprPath(const SourcePath & path); struct InvalidPathError : EvalError { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 81e94848a..2b858ee98 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -218,7 +218,7 @@ static Flake getFlake( throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1)); @@ -731,7 +731,7 @@ void callFlake(EvalState & state, state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , "/"), **state.vCallFlake); + , CanonPath::root), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 97e615c37..8b67665db 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,7 +31,7 @@ namespace nix { EvalState & state; SymbolTable & symbols; Expr * result; - Path basePath; + SourcePath basePath; PosTable::Origin origin; std::optional error; }; @@ -509,7 +509,7 @@ string_parts_interpolated path_start : PATH { - Path path(absPath({$1.p, $1.l}, data->basePath)); + Path path(absPath({$1.p, $1.l}, data->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; @@ -651,7 +651,7 @@ Expr * EvalState::parse( char * text, size_t length, Pos::Origin origin, - Path basePath, + const SourcePath & basePath, std::shared_ptr & staticEnv) { yyscan_t scanner; @@ -675,48 +675,36 @@ Expr * EvalState::parse( } -Path resolveExprPath(Path path) +SourcePath resolveExprPath(const SourcePath & path) { - assert(path[0] == '/'); - - unsigned int followCount = 0, maxFollow = 1024; - /* If `path' is a symlink, follow it. This is so that relative path references work. */ - struct stat st; - while (true) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - st = lstat(path); - if (!S_ISLNK(st.st_mode)) break; - path = absPath(readLink(path), dirOf(path)); - } + auto path2 = path.resolveSymlinks(); /* If `path' refers to a directory, append `/default.nix'. */ - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); + if (path2.lstat().type == InputAccessor::tDirectory) + return path2 + "default.nix"; - return path; + return path2; } -Expr * EvalState::parseExprFromFile(const Path & path) +Expr * EvalState::parseExprFromFile(const SourcePath & path) { return parseExprFromFile(path, staticBaseEnv); } -Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr & staticEnv) +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto buffer = readFile(path); - // readFile should have left some extra space for terminators + auto buffer = path.readFile(); + // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv); + return parse(buffer.data(), buffer.size(), path.path.abs(), path.parent(), staticEnv); } -Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr & staticEnv) +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr & staticEnv) { auto s = make_ref(std::move(s_)); s->append("\0\0", 2); @@ -724,7 +712,7 @@ Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std } -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) { return parseExprFromString(std::move(s), basePath, staticBaseEnv); } @@ -737,7 +725,7 @@ Expr * EvalState::parseStdin() // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); } @@ -757,13 +745,13 @@ void EvalState::addToSearchPath(const std::string & s) } -Path EvalState::findFile(const std::string_view path) +SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); } -Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) { for (auto & i : searchPath) { std::string suffix; @@ -779,11 +767,11 @@ Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, c auto r = resolveSearchPathElem(i); if (!r.first) continue; Path res = r.second + suffix; - if (pathExists(res)) return canonPath(res); + if (pathExists(res)) return CanonPath(canonPath(res)); } if (hasPrefix(path, "nix/")) - return concatStrings(corepkgsPrefix, path.substr(4)); + return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc new file mode 100644 index 000000000..1d690b722 --- /dev/null +++ b/src/libexpr/paths.cc @@ -0,0 +1,10 @@ +#include "eval.hh" + +namespace nix { + +SourcePath EvalState::rootPath(CanonPath path) +{ + return std::move(path); +} + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 72faeada8..c77d45bd3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -110,7 +110,7 @@ struct RealisePathFlags { bool checkForPureEval = true; }; -static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { PathSet context; @@ -119,7 +119,7 @@ static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const Re try { StringMap rewrites = state.realiseContext(context); - auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); return flags.checkForPureEval ? state.checkSourcePath(realPath) @@ -166,13 +166,14 @@ static void mkOutputString( static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { auto path = realisePath(state, pos, vPath); + auto path2 = path.path.abs(); // FIXME auto isValidDerivationInStore = [&]() -> std::optional { - if (!state.store->isStorePath(path)) + if (!state.store->isStorePath(path2)) return std::nullopt; - auto storePath = state.store->parseStorePath(path); - if (!(state.store->isValidPath(storePath) && isDerivation(path))) + auto storePath = state.store->parseStorePath(path2); + if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; }; @@ -181,7 +182,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v auto storePath = *optStorePath; Derivation drv = state.store->readDerivation(storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path2, {"=" + path2}); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); @@ -198,7 +199,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **state.vImportedDrvToDerivation); + , CanonPath::root), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -206,10 +207,10 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path == corepkgsPrefix + "fetchurl.nix") { + else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , "/"), v); + , CanonPath::root), v); } else { @@ -330,7 +331,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); - void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); @@ -378,7 +379,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - parsed = state.parseExprFromString(std::move(output), "/"); + parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root)); } catch (Error & e) { e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program); throw; @@ -585,7 +586,7 @@ struct CompareValues case nString: return strcmp(v1->string.s, v2->string.s) < 0; case nPath: - return strcmp(v1->path, v2->path) < 0; + return strcmp(v1->_path, v2->_path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -1428,8 +1429,8 @@ static RegisterPrimOp primop_placeholder({ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); - v.mkString(canonPath(path), context); + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); + v.mkString(path.path.abs(), context); } static RegisterPrimOp primop_toPath({ @@ -1459,21 +1460,22 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")); + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path)) path = canonPath(path, true); - if (!state.store->isInStore(path)) + if (!state.store->isStorePath(path.abs())) + path = CanonPath(canonPath(path.abs(), true)); + if (!state.store->isInStore(path.abs())) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("path '%1%' is not in the Nix store", path), .errPos = state.positions[pos] })); - auto path2 = state.store->toStorePath(path).first; + auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); context.insert(state.store->printStorePath(path2)); - v.mkString(path, context); + v.mkString(path.abs(), context); } static RegisterPrimOp primop_storePath({ @@ -1504,7 +1506,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { - v.mkBool(pathExists(state.checkSourcePath(path))); + v.mkBool(state.checkSourcePath(path).pathExists()); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1551,11 +1553,17 @@ static RegisterPrimOp primop_baseNameOf({ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto path = state.coerceToString(pos, *args[0], context, - "while evaluating the first argument passed to builtins.dirOf", + state.forceValue(*args[0], pos); + if (args[0]->type() == nPath) { + auto path = args[0]->path(); + v.mkPath(path.path.isRoot() ? path : path.parent()); + } else { + auto path = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to 'builtins.dirOf'", false, false); - auto dir = dirOf(*path); - if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); + auto dir = dirOf(*path); + v.mkString(dir, context); + } } static RegisterPrimOp primop_dirOf({ @@ -1573,13 +1581,13 @@ static RegisterPrimOp primop_dirOf({ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - auto s = readFile(path); + auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); StorePathSet refs; - if (state.store->isInStore(path)) { + if (state.store->isInStore(path.path.abs())) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1660,7 +1668,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashFile(*ht, path).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1674,26 +1682,20 @@ static RegisterPrimOp primop_hashFile({ .fun = prim_hashFile, }); - -/* Stringize a directory entry enum. Used by `readFileType' and `readDir'. */ -static const char * dirEntTypeToString(unsigned char dtType) +static std::string_view fileTypeToString(InputAccessor::Type type) { - /* Enum DT_(DIR|LNK|REG|UNKNOWN) */ - switch(dtType) { - case DT_REG: return "regular"; break; - case DT_DIR: return "directory"; break; - case DT_LNK: return "symlink"; break; - default: return "unknown"; break; - } - return "unknown"; /* Unreachable */ + return + type == InputAccessor::Type::tRegular ? "regular" : + type == InputAccessor::Type::tDirectory ? "directory" : + type == InputAccessor::Type::tSymlink ? "symlink" : + "unknown"; } - static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); /* Retrieve the directory entry type and stringize it. */ - v.mkString(dirEntTypeToString(getFileType(path))); + v.mkString(fileTypeToString(path.lstat().type)); } static RegisterPrimOp primop_readFileType({ @@ -1714,8 +1716,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // Retrieve directory entries for all nodes in a directory. // This is similar to `getFileType` but is optimized to reduce system calls // on many systems. - DirEntries entries = readDirectory(path); - + auto entries = path.readDirectory(); auto attrs = state.buildBindings(entries.size()); // If we hit unknown directory entry types we may need to fallback to @@ -1724,22 +1725,21 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // `builtins.readFileType` application. Value * readFileType = nullptr; - for (auto & ent : entries) { - auto & attr = attrs.alloc(ent.name); - if (ent.type == DT_UNKNOWN) { + for (auto & [name, type] : entries) { + auto & attr = attrs.alloc(name); + if (!type) { // Some filesystems or operating systems may not be able to return // detailed node info quickly in this case we produce a thunk to // query the file type lazily. auto epath = state.allocValue(); - Path path2 = path + "/" + ent.name; - epath->mkString(path2); + epath->mkPath(path + name); if (!readFileType) readFileType = &state.getBuiltin("readFileType"); attr.mkApp(readFileType, epath); } else { // This branch of the conditional is much more likely. // Here we just stringize the directory entry type. - attr.mkString(dirEntTypeToString(ent.type)); + attr.mkString(fileTypeToString(*type)); } } @@ -2045,7 +2045,7 @@ static RegisterPrimOp primop_toFile({ static void addPath( EvalState & state, const PosIdx pos, - const std::string & name, + std::string_view name, Path path, Value * filterFun, FileIngestionMethod method, @@ -2073,7 +2073,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(path); + : state.checkSourcePath(CanonPath(path)).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2120,9 +2120,10 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + auto path = state.coerceToPath(pos, *args[1], context, + "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2182,18 +2183,19 @@ static RegisterPrimOp primop_filterSource({ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.path"); - Path path; + std::optional path; std::string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; PathSet context; + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); + for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; if (n == "path") - path = state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the `path` attribute passed to builtins.path"); + path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); else if (n == "filter") @@ -2208,15 +2210,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value .errPos = state.positions[attr.pos] })); } - if (path.empty()) + if (!path) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), .errPos = state.positions[pos] })); if (name.empty()) - name = baseNameOf(path); + name = path->baseName(); - addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); + addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -4151,7 +4153,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationNixPath, {CanonPath::root}, staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 69c932f05..b8e65aafe 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -28,7 +28,7 @@ namespace nix { } Value eval(std::string input, bool forceValue = true) { Value v; - Expr * e = state.parseExprFromString(input, ""); + Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); assert(e); state.eval(e, v); if (forceValue) diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 3e5504f71..331a5ead6 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -12,7 +12,7 @@ libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ $(wildcard $(d)/value/*.cc) -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests +libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index c35c876e3..d40a77302 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -36,9 +36,10 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.store->printStorePath(state.copyPathToStore(context, v.path)); + out = state.store->printStorePath( + state.copyPathToStore(context, v.path())); else - out = v.path; + out = v.path().path.abs(); break; case nNull: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 341c8922f..a9fcb803e 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -78,7 +78,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; case nPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + doc.writeEmptyElement("path", singletonAttrs("value", v.path().to_string())); break; case nNull: diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index bfae4ee94..eb6f56d07 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -5,6 +5,7 @@ #include "symbol-table.hh" #include "value/context.hh" +#include "input-accessor.hh" #if HAVE_BOEHMGC #include @@ -171,7 +172,7 @@ public: const char * * context; // must be in sorted order } string; - const char * path; + const char * _path; Bindings * attrs; struct { size_t size; @@ -251,15 +252,20 @@ public: void mkStringMove(const char * s, const PathSet & context); - inline void mkPath(const char * s) + inline void mkString(const Symbol & s) + { + mkString(((const std::string &) s).c_str()); + } + + void mkPath(const SourcePath & path); + + inline void mkPath(const char * path) { clearValue(); internalType = tPath; - path = s; + _path = path; } - void mkPath(std::string_view s); - inline void mkNull() { clearValue(); @@ -400,6 +406,18 @@ public: auto begin = listElems(); return ConstListIterable { begin, begin + listSize() }; } + + SourcePath path() const + { + assert(internalType == tPath); + return SourcePath{CanonPath(_path)}; + } + + std::string_view str() const + { + assert(internalType == tString); + return std::string_view(string.s); + } }; diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc new file mode 100644 index 000000000..f9909c218 --- /dev/null +++ b/src/libfetchers/input-accessor.cc @@ -0,0 +1,100 @@ +#include "input-accessor.hh" +#include "store-api.hh" + +namespace nix { + +std::ostream & operator << (std::ostream & str, const SourcePath & path) +{ + str << path.to_string(); + return str; +} + +std::string_view SourcePath::baseName() const +{ + return path.baseName().value_or("source"); +} + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return std::move(*p); +} + +InputAccessor::Stat SourcePath::lstat() const +{ + auto st = nix::lstat(path.abs()); + return InputAccessor::Stat { + .type = + S_ISREG(st.st_mode) ? InputAccessor::tRegular : + S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : + S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : + InputAccessor::tMisc, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + }; +} + +std::optional SourcePath::maybeLstat() const +{ + // FIXME: merge these into one operation. + if (!pathExists()) + return {}; + return lstat(); +} + +InputAccessor::DirEntries SourcePath::readDirectory() const +{ + InputAccessor::DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = InputAccessor::Type::tRegular; break; + case DT_LNK: type = InputAccessor::Type::tSymlink; break; + case DT_DIR: type = InputAccessor::Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +StorePath SourcePath::fetchToStore( + ref store, + std::string_view name, + PathFilter * filter, + RepairFlag repair) const +{ + return + settings.readOnlyMode + ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first + : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); +} + +SourcePath SourcePath::resolveSymlinks() const +{ + SourcePath res(CanonPath::root); + + int linksAllowed = 1024; + + for (auto & component : path) { + res.path.push(component); + while (true) { + if (auto st = res.maybeLstat()) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", path); + if (st->type != InputAccessor::tSymlink) break; + auto target = res.readLink(); + if (hasPrefix(target, "/")) + res = CanonPath(target); + else { + res.path.pop(); + res.path.extend(CanonPath(target)); + } + } else + break; + } + } + + return res; +} + +} diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh new file mode 100644 index 000000000..23c510d4d --- /dev/null +++ b/src/libfetchers/input-accessor.hh @@ -0,0 +1,149 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "archive.hh" +#include "canon-path.hh" +#include "repair-flag.hh" + +namespace nix { + +class StorePath; +class Store; + +struct InputAccessor +{ + enum Type { tRegular, tSymlink, tDirectory, tMisc }; + + struct Stat + { + Type type = tMisc; + //uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + }; + + typedef std::optional DirEntry; + + typedef std::map DirEntries; +}; + +/** + * An abstraction for accessing source files during + * evaluation. Currently, it's just a wrapper around `CanonPath` that + * accesses files in the regular filesystem, but in the future it will + * support fetching files in other ways. + */ +struct SourcePath +{ + CanonPath path; + + SourcePath(CanonPath path) + : path(std::move(path)) + { } + + std::string_view baseName() const; + + /** + * Construct the parent of this `SourcePath`. Aborts if `this` + * denotes the root. + */ + SourcePath parent() const; + + /** + * If this `SourcePath` denotes a regular file (not a symlink), + * return its contents; otherwise throw an error. + */ + std::string readFile() const + { return nix::readFile(path.abs()); } + + /** + * Return whether this `SourcePath` denotes a file (of any type) + * that exists + */ + bool pathExists() const + { return nix::pathExists(path.abs()); } + + /** + * Return stats about this `SourcePath`, or throw an exception if + * it doesn't exist. + */ + InputAccessor::Stat lstat() const; + + /** + * Return stats about this `SourcePath`, or std::nullopt if it + * doesn't exist. + */ + std::optional maybeLstat() const; + + /** + * If this `SourcePath` denotes a directory (not a symlink), + * return its directory entries; otherwise throw an error. + */ + InputAccessor::DirEntries readDirectory() const; + + /** + * If this `SourcePath` denotes a symlink, return its target; + * otherwise throw an error. + */ + std::string readLink() const + { return nix::readLink(path.abs()); } + + /** + * Dump this `SourcePath` to `sink` as a NAR archive. + */ + void dumpPath( + Sink & sink, + PathFilter & filter = defaultPathFilter) const + { return nix::dumpPath(path.abs(), sink, filter); } + + /** + * Copy this `SourcePath` to the Nix store. + */ + StorePath fetchToStore( + ref store, + std::string_view name = "source", + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair) const; + + /** + * Return the location of this path in the "real" filesystem, if + * it has a physical location. + */ + std::optional getPhysicalPath() const + { return path; } + + std::string to_string() const + { return path.abs(); } + + SourcePath operator + (const CanonPath & x) const + { return {path + x}; } + + SourcePath operator + (std::string_view c) const + { return {path + c}; } + + bool operator == (const SourcePath & x) const + { + return path == x.path; + } + + bool operator != (const SourcePath & x) const + { + return path != x.path; + } + + bool operator < (const SourcePath & x) const + { + return path < x.path; + } + + /** + * Resolve any symlinks in this `SourcePath` (including its + * parents). The result is a `SourcePath` in which no element is a + * symlink. + */ + SourcePath resolveSymlinks() const; +}; + +std::ostream & operator << (std::ostream & str, const SourcePath & path); + +} diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index ddf6db6d1..040464532 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -13,6 +13,11 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root) : path(absPath((Path) raw, root.abs())) { } +CanonPath CanonPath::fromCwd(std::string_view path) +{ + return CanonPath(unchecked_t(), absPath((Path) path)); +} + std::optional CanonPath::parent() const { if (isRoot()) return std::nullopt; diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 76e48c4f2..30f43b5a4 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -46,6 +46,8 @@ public: : path(std::move(path)) { } + static CanonPath fromCwd(std::string_view path = "."); + static CanonPath root; /** diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index bc7e7eb18..251f00edf 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -289,7 +289,7 @@ static void main_nix_build(int argc, char * * argv) else for (auto i : left) { if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), absPath("."))); + exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath(CanonPath::fromCwd()))); else { auto absolute = i; try { @@ -385,7 +385,9 @@ static void main_nix_build(int argc, char * * argv) if (!shell) { try { - auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); + auto expr = state->parseExprFromString( + "(import {}).bashInteractive", + state->rootPath(CanonPath::fromCwd())); Value v; state->eval(expr, v); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f076ffdb0..0081bab25 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -44,7 +44,7 @@ typedef enum { struct InstallSourceInfo { InstallSourceType type; - Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + std::shared_ptr nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ Path profile; /* for srcProfile */ std::string systemFilter; /* for srcNixExprDrvs */ Bindings * autoArgs; @@ -92,9 +92,11 @@ static bool parseInstallSourceOptions(Globals & globals, } -static bool isNixExpr(const Path & path, struct stat & st) +static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { - return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); + return + st.type == InputAccessor::tRegular + || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists()); } @@ -102,10 +104,10 @@ static constexpr size_t maxAttrs = 1024; static void getAllExprs(EvalState & state, - const Path & path, StringSet & seen, BindingsBuilder & attrs) + const SourcePath & path, StringSet & seen, BindingsBuilder & attrs) { StringSet namesSorted; - for (auto & i : readDirectory(path)) namesSorted.insert(i.name); + for (auto & [name, _] : path.readDirectory()) namesSorted.insert(name); for (auto & i : namesSorted) { /* Ignore the manifest.nix used by profiles. This is @@ -113,13 +115,16 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - Path path2 = path + "/" + i; + SourcePath path2 = path + i; - struct stat st; - if (stat(path2.c_str(), &st) == -1) + InputAccessor::Stat st; + try { + st = path2.resolveSymlinks().lstat(); + } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr + } - if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + if (isNixExpr(path2, st) && (st.type != InputAccessor::tRegular || hasSuffix(path2.baseName(), ".nix"))) { /* Strip off the `.nix' filename suffix (if applicable), otherwise the attribute cannot be selected with the `-A' option. Useful if you want to stick a Nix @@ -129,21 +134,20 @@ static void getAllExprs(EvalState & state, attrName = std::string(attrName, 0, attrName.size() - 4); if (!seen.insert(attrName).second) { std::string suggestionMessage = ""; - if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) { + if (path2.path.abs().find("channels") != std::string::npos && path.path.abs().find("channels") != std::string::npos) suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName); - } printError("warning: name collision in input Nix expressions, skipping '%1%'" "%2%", path2, suggestionMessage); continue; } /* Load the expression on demand. */ auto vArg = state.allocValue(); - vArg->mkString(path2); + vArg->mkString(path2.path.abs()); if (seen.size() == maxAttrs) throw Error("too many Nix expressions in directory '%1%'", path); attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg); } - else if (S_ISDIR(st.st_mode)) + else if (st.type == InputAccessor::tDirectory) /* `path2' is a directory (with no default.nix in it); recurse into it. */ getAllExprs(state, path2, seen, attrs); @@ -152,11 +156,9 @@ static void getAllExprs(EvalState & state, -static void loadSourceExpr(EvalState & state, const Path & path, Value & v) +static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v) { - struct stat st; - if (stat(path.c_str(), &st) == -1) - throw SysError("getting information about '%1%'", path); + auto st = path.resolveSymlinks().lstat(); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -167,7 +169,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - else if (S_ISDIR(st.st_mode)) { + else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); attrs.alloc("_combineChannels").mkList(0); StringSet seen; @@ -179,7 +181,7 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) } -static void loadDerivations(EvalState & state, Path nixExprPath, +static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, std::string systemFilter, Bindings & autoArgs, const std::string & pathPrefix, DrvInfos & elems) { @@ -390,7 +392,7 @@ static void queryInstSources(EvalState & state, /* Load the derivations from the (default or specified) Nix expression. */ DrvInfos allElems; - loadDerivations(state, instSource.nixExprPath, + loadDerivations(state, *instSource.nixExprPath, instSource.systemFilter, *instSource.autoArgs, "", allElems); elems = filterBySelector(state, allElems, args, newestOnly); @@ -407,10 +409,10 @@ static void queryInstSources(EvalState & state, case srcNixExprs: { Value vArg; - loadSourceExpr(state, instSource.nixExprPath, vArg); + loadSourceExpr(state, *instSource.nixExprPath, vArg); for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, absPath(".")); + Expr * eFun = state.parseExprFromString(i, state.rootPath(CanonPath::fromCwd())); Value vFun, vTmp; state.eval(eFun, vFun); vTmp.mkApp(&vFun, &vArg); @@ -462,7 +464,7 @@ static void queryInstSources(EvalState & state, case srcAttrPath: { Value vRoot; - loadSourceExpr(state, instSource.nixExprPath, vRoot); + loadSourceExpr(state, *instSource.nixExprPath, vRoot); for (auto & i : args) { Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); @@ -1030,7 +1032,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) installedElems = queryInstalled(*globals.state, globals.profile); if (source == sAvailable || compareVersions) - loadDerivations(*globals.state, globals.instSource.nixExprPath, + loadDerivations(*globals.state, *globals.instSource.nixExprPath, globals.instSource.systemFilter, *globals.instSource.autoArgs, attrPath, availElems); @@ -1395,22 +1397,20 @@ static int main_nix_env(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - { - Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; - globals.instSource.nixExprPath = nixExprPath; - } globals.instSource.systemFilter = "*"; - if (!pathExists(globals.instSource.nixExprPath)) { + Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + + if (!pathExists(nixExprPath)) { try { - createDirs(globals.instSource.nixExprPath); + createDirs(nixExprPath); replaceSymlink( defaultChannelsDir(), - globals.instSource.nixExprPath + "/channels"); + nixExprPath + "/channels"); if (getuid() != 0) replaceSymlink( rootChannelsDir(), - globals.instSource.nixExprPath + "/channels_root"); + nixExprPath + "/channels_root"); } catch (Error &) { } } @@ -1517,8 +1517,10 @@ static int main_nix_env(int argc, char * * argv) globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); globals.state->repair = repair; - if (file != "") - globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + globals.instSource.nixExprPath = std::make_shared( + file != "" + ? lookupFileArg(*globals.state, file) + : globals.state->rootPath(CanonPath(nixExprPath))); globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 745e9e174..f70381bf2 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -19,10 +19,10 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) DrvInfos elems; if (pathExists(userEnv + "/manifest.json")) throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - Path manifestFile = userEnv + "/manifest.nix"; + auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(manifestFile, v); + state.evalFile(state.rootPath(CanonPath(manifestFile)), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } @@ -114,7 +114,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Value envBuilder; state.eval(state.parseExprFromString( #include "buildenv.nix.gen.hh" - , "/"), envBuilder); + , state.rootPath(CanonPath::root)), envBuilder); /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6b5ba595d..2b381288a 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -168,9 +168,11 @@ static int main_nix_instantiate(int argc, char * * argv) if (findFile) { for (auto & i : files) { - Path p = state->findFile(i); - if (p == "") throw Error("unable to find '%1%'", i); - std::cout << p << std::endl; + auto p = state->findFile(i); + if (auto fn = p.getPhysicalPath()) + std::cout << fn->abs() << std::endl; + else + throw Error("'%s' has no physical path", p); } return 0; } @@ -184,7 +186,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs - ? state->parseExprFromString(i, absPath(".")) + ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 43db5150c..c7af9c92c 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -66,7 +66,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (apply) { auto vApply = state->allocValue(); - state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + state->eval(state->parseExprFromString(*apply, state->rootPath(CanonPath::fromCwd())), *vApply); auto vRes = state->allocValue(); state->callFunction(*vApply, *v, *vRes, noPos); v = vRes; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cd4ee5921..a6c6b947f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -440,8 +440,8 @@ struct CmdFlakeCheck : FlakeCommand if (attr->name == state->symbols.create("path")) { PathSet context; auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); - if (!store->isInStore(path)) - throw Error("template '%s' has a bad 'path' attribute"); + if (!path.pathExists()) + throw Error("template '%s' refers to a non-existent path '%s'", attrPath, path); // TODO: recursively check the flake in 'path'. } } else diff --git a/src/nix/main.cc b/src/nix/main.cc index 54c920b4e..df0a25214 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -197,14 +197,14 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , "/"), *vGenerateManpage); + , CanonPath::root), *vGenerateManpage); auto vUtils = state.allocValue(); state.cacheFile( - "/utils.nix", "/utils.nix", + CanonPath("/utils.nix"), CanonPath("/utils.nix"), state.parseExprFromString( #include "utils.nix.gen.hh" - , "/"), + , CanonPath::root), *vUtils); auto vDump = state.allocValue(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 51c8a3319..039608d48 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -27,7 +27,10 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) Value vMirrors; // FIXME: use nixpkgs flake - state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.eval(state.parseExprFromString( + "import ", + state.rootPath(CanonPath::root)), + vMirrors); state.forceAttrs(vMirrors, noPos, "while evaluating the set of all mirrors"); auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); @@ -192,9 +195,11 @@ static int main_nix_prefetch_url(int argc, char * * argv) throw UsageError("you must specify a URL"); url = args[0]; } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; - state->evalFile(path, vRoot); + state->evalFile( + resolveExprPath( + lookupFileArg(*state, args.empty() ? "." : args[0])), + vRoot); Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); state->forceAttrs(v, noPos, "while evaluating the source attribute to prefetch"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17796d6b8..9185ba407 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -140,7 +140,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto state = std::make_unique(Strings(), store); auto v = state->allocValue(); - state->eval(state->parseExprFromString(res.data, "/no-such-path"), *v); + state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 8182a6a83..40350aa96 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr +libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr -I src/libfetchers From a9759407e55fb02c6e306fdd9fcedd821e465024 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Apr 2023 15:25:06 +0200 Subject: [PATCH 045/114] Origin: Use SourcePath --- src/libcmd/repl.cc | 4 ++-- src/libexpr/eval.cc | 17 +++++++++-------- src/libexpr/eval.hh | 5 ++--- src/libexpr/flake/flake.cc | 2 +- src/libexpr/nixexpr.cc | 6 +++--- src/libexpr/nixexpr.hh | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops.cc | 3 +-- src/libexpr/value-to-xml.cc | 4 ++-- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 9002fa555..c7fd0db91 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -594,8 +594,8 @@ bool NixRepl::processLine(std::string line) return {path, 0}; } else if (v.isLambda()) { auto pos = state->positions[v.lambda.fun->pos]; - if (auto path = std::get_if(&pos.origin)) - return {SourcePath(CanonPath(*path)), pos.line}; + if (auto path = std::get_if(&pos.origin)) + return {*path, pos.line}; else throw Error("'%s' cannot be shown in an editor", pos); } else { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3b2877602..dec123b69 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -499,6 +499,7 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -991,9 +992,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, PosIdx p) { auto pos = positions[p]; - if (auto path = std::get_if(&pos.origin)) { + if (auto path = std::get_if(&pos.origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(*path); + attrs.alloc(sFile).mkString(path->path.abs()); attrs.alloc(sLine).mkInt(pos.line); attrs.alloc(sColumn).mkInt(pos.column); v.mkAttrs(attrs); @@ -1373,8 +1374,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } catch (Error & e) { if (pos2) { auto pos2r = state.positions[pos2]; - auto origin = std::get_if(&pos2r.origin); - if (!(origin && *origin == state.derivationNixPath)) + auto origin = std::get_if(&pos2r.origin); + if (!(origin && *origin == state.derivationInternal)) state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); } @@ -2417,8 +2418,8 @@ void EvalState::printStats() else obj["name"] = nullptr; if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = *path; + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); obj["line"] = pos.line; obj["column"] = pos.column; } @@ -2432,8 +2433,8 @@ void EvalState::printStats() for (auto & i : attrSelects) { json obj = json::object(); if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = *path; + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); obj["line"] = pos.line; obj["column"] = pos.column; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1e8f8391c..b9578321f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -127,8 +127,6 @@ public: SymbolTable symbols; PosTable positions; - static inline std::string derivationNixPath = "//builtin/derivation.nix"; - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, @@ -139,7 +137,6 @@ public: sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, sPrefix, sOutputSpecified; - Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they already exist there. */ @@ -151,6 +148,8 @@ public: Bindings emptyBindings; + const SourcePath derivationInternal; + /* Store used to materialise .drv files. */ const ref store; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2b858ee98..4c571fd7d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -220,7 +220,7 @@ static Flake getFlake( Value vInfo; state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..2c9d5754e 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -31,9 +31,9 @@ struct PosAdapter : AbstractPos // Get rid of the null terminators added by the parser. return std::string(s.source->c_str()); }, - [](const Path & path) -> std::optional { + [](const SourcePath & path) -> std::optional { try { - return readFile(path); + return path.readFile(); } catch (Error &) { return std::nullopt; } @@ -47,7 +47,7 @@ struct PosAdapter : AbstractPos [&](const Pos::none_tag &) { out << "«none»"; }, [&](const Pos::Stdin &) { out << "«stdin»"; }, [&](const Pos::String & s) { out << "«string»"; }, - [&](const Path & path) { out << path; } + [&](const SourcePath & path) { out << path; } }, origin); } }; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4079a7b24..d70280582 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -32,7 +32,7 @@ struct Pos struct Stdin { ref source; }; struct String { ref source; }; - typedef std::variant Origin; + typedef std::variant Origin; Origin origin; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8b67665db..4d981712a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -700,7 +700,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr(&pos.origin)) - xmlAttrs["path"] = *path; + if (auto path = std::get_if(&pos.origin)) + xmlAttrs["path"] = path->path.abs(); xmlAttrs["line"] = fmt("%1%", pos.line); xmlAttrs["column"] = fmt("%1%", pos.column); } From 8a7790f46aaf608bd7ed4ec6042b99bd36d7118e Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Wed, 5 Apr 2023 20:10:11 -0700 Subject: [PATCH 046/114] Expand documentation for `experimental-features` Adds examples and additional information to the `impure-derivations`, `recursive-nix`, and `no-url-literals` experimental feature documentation. --- src/libutil/experimental-features.cc | 111 +++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 14 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 010ab1d68..b7879ff69 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -17,9 +17,9 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::CaDerivations, .name = "ca-derivations", .description = R"( - Allow derivations to be content-addressed in order to prevent rebuilds - when changes to the derivation do not result in changes to the - derivation's output. See + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) for details. )", @@ -28,17 +28,36 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ImpureDerivations, .name = "impure-derivations", .description = R"( - Allows derivations to produce non-fixed outputs by setting the `__impure` - derivation attribute to `true`. See [these release - notes](../release-notes/rl-2.8.md) for an example. + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). )", }, { .tag = Xp::Flakes, .name = "flakes", .description = R"( - Enable flakes. See the manual entry for - [`nix flake`](../command-ref/new-cli/nix3-flake.md) for details. + Enable flakes. See the manual entry for [`nix + flake`](../command-ref/new-cli/nix3-flake.md) for details. )", }, { @@ -53,18 +72,82 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::RecursiveNix, .name = "recursive-nix", .description = R"( - Allow Nix derivations to call Nix in order to recursively build derivations. - See [this - commit](https://github.com/edolstra/nix/commit/1a27aa7d64ffe6fc36cfca4d82bdf51c4d8cf717) - for more info. + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${}"; + } + '' + hello=$(nix-build -E '(import {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. )", }, { .tag = Xp::NoUrlLiterals, .name = "no-url-literals", .description = R"( - Disallows unquoted URLs as part of the Nix language syntax. See [RFC - 45](https://github.com/NixOS/rfcs/pull/45) for more info. + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://bar + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. )", }, { From e399cb49c245de909417cfbe39a4c64fe0632653 Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Thu, 6 Apr 2023 15:02:19 -0700 Subject: [PATCH 047/114] Fix typo in `no-url-literals` experimental feature docs Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b7879ff69..aa3367639 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -136,7 +136,7 @@ constexpr std::array xpFeatureDetails = {{ at «string»:1:1: - 1| http://bar + 1| http://foo | ^ ``` From 6c4049b38a2e694d904b4bc4f3525efe68f2f611 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 11:37:02 -0400 Subject: [PATCH 048/114] Link the new general documentation on xp features on the setting --- src/libutil/config.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 8b0fe6555..b21f8a422 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -385,6 +385,8 @@ struct ExperimentalFeatureSettings : Config { Experimental features available: {{#include experimental-features.md}} + + Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. )"}; /** From bc192a95ef4c777d9ca0ad025e0fb2b7f18879f7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 12:24:11 -0400 Subject: [PATCH 049/114] Describe active experimental features in the contributing guide They are put in the manual separate pages under the new overarching description of experimental features. The settings page just lists the valid experimental feature names (so people know what a valid setting entry looks like), with links to those pages. It doesn't attempt to describe each experimental feature as that is too much information for the configuration settings section. --- .gitignore | 3 ++- doc/manual/generate-xp-features-shortlist.nix | 9 ++++++++ doc/manual/generate-xp-features.nix | 22 ++++++++++++++----- doc/manual/local.mk | 14 ++++++++---- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/utils.nix | 3 +++ src/libutil/config.hh | 2 +- src/libutil/experimental-features.cc | 2 +- 8 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 doc/manual/generate-xp-features-shortlist.nix diff --git a/.gitignore b/.gitignore index 53442751f..ffaf52be8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,7 +23,8 @@ perl/Makefile.config /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md -/doc/manual/src/command-ref/experimental-features.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-features /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix new file mode 100644 index 000000000..b2095bc27 --- /dev/null +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -0,0 +1,9 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/contributing/experimental-features/${name}.md) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index db1ba6092..ff64edcf7 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,11 +1,21 @@ +xps: + with builtins; with import ./utils.nix; let - showExperimentalFeature = name: doc: - squash '' - - [`${name}`](#xp-feature-${name}) + makePage = { name, value }: + { + name = "${name}.md"; + inherit value; + feature = name; + }; - ${indent " " doc} - ''; -in xps: indent " " (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) + featurePages = map makePage (attrsToList xps); + + tableOfContents = let + showEntry = page: + " - [${page.feature}](contributing/experimental-features/${page.name})"; + in concatStringsSep "\n" (map showEntry featurePages) + "\n"; + +in (listToAttrs featurePages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 0d4b9d640..ae55ae4a9 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -81,10 +81,11 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features @cp $< $@ @$(call process-includes,$@,$@) @@ -93,7 +94,7 @@ $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,11 +107,16 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-features: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + $(d)/xp-features.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ @@ -154,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5bf274550..298644044 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -96,6 +96,7 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Experimental Features](contributing/experimental-features.md) +{{#include ./contributing/experimental-features/SUMMARY.md}} - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index f78e6bb02..82544935a 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -5,6 +5,9 @@ rec { concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + replaceStringsRec = from: to: string: # recursively replace occurrences of `from` with `to` within `string` # example: diff --git a/src/libutil/config.hh b/src/libutil/config.hh index b21f8a422..6baba0167 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -384,7 +384,7 @@ struct ExperimentalFeatureSettings : Config { Experimental features available: - {{#include experimental-features.md}} + {{#include experimental-features-shortlist.md}} Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. )"}; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index aa3367639..1cd0dbb79 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -57,7 +57,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "flakes", .description = R"( Enable flakes. See the manual entry for [`nix - flake`](../command-ref/new-cli/nix3-flake.md) for details. + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, { From 214f1d67910e491e0bd7e1f09f0623fdffd82d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 7 Apr 2023 09:16:40 +0200 Subject: [PATCH 050/114] Rename and protect `BufferedSink::write` The `write` name is ambiguous and could lead to some funny bugs like https://github.com/NixOS/nix/pull/8173#issuecomment-1500009480. So rename it to the more explicit `writeUnbuffered`. Besides, this method shouldn't be (and isn't) used outside of the class implementation, so mark it `protected`. This makes it more symetrical to `BufferedSource` which uses a `protected readUnbuffered` method. --- src/libutil/compression.cc | 6 +++--- src/libutil/compression.hh | 2 +- src/libutil/hash.cc | 2 +- src/libutil/hash.hh | 2 +- src/libutil/serialise.cc | 6 +++--- src/libutil/serialise.hh | 8 +++++--- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 89180e7a7..ba0847cde 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); if (result <= 0) check(result); @@ -136,7 +136,7 @@ struct NoneSink : CompressionSink warn("requested compression level '%d' not supported by compression method 'none'", level); } void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void writeUnbuffered(std::string_view data) override { nextSink(data); } }; struct BrotliDecompressionSink : ChunkedCompressionSink diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 3892831c2..4e53a7b3c 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -12,7 +12,7 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..0c8f5727d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -343,7 +343,7 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); update(ht, *ctx, data); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index be1fdba2a..ae3ee40f4 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -197,7 +197,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..62d426e5c 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data) buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,7 +48,7 @@ FdSink::~FdSink() } -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); try { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2cf527023..cdb991036 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -53,7 +53,9 @@ struct BufferedSink : virtual Sink void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; @@ -133,7 +135,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -520,7 +522,7 @@ struct FramedSink : nix::BufferedSink } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ From 6960de73affeb7a6d061837b1088937a0295dd93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 7 Apr 2023 13:08:32 +0200 Subject: [PATCH 051/114] Typo --- src/libexpr/flake/lockfile.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 4616a77f6..868ab89ad 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -67,7 +67,7 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't syntethise it (like gcc 8.2.2 + // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 // that is still the default on aarch64-linux) bool operator !=(const LockFile & other) const; From 6e0b7109abb40ded327b15599b29f861d9acb3c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 13:34:32 +0100 Subject: [PATCH 052/114] Move OpenSSL init to initLibUtil Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 22 +--------------------- src/libutil/hash.cc | 23 +++++++++++++++++++++++ src/libutil/util.cc | 4 ++++ src/libutil/util.hh | 3 +++ 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 37664c065..2ed310cba 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -115,22 +115,6 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - static std::once_flag dns_resolve_flag; static void preloadNSS() { @@ -177,11 +161,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif + initLibUtil(); if (sodium_init() == -1) throw Error("could not initialise libsodium"); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..9df8bcfb4 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -16,6 +17,28 @@ namespace nix { +#if OPENSSL_VERSION_NUMBER < 0x10101000L +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector opensslLocks; + +static void opensslLockCallback(int mode, int type, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} +#endif + +void initOpenSSL() { +#if OPENSSL_VERSION_NUMBER < 0x10101000L + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); +#endif +} static size_t regularHashSize(HashType type) { switch (type) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 843a10eab..0099f7ebc 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -47,6 +47,10 @@ extern char * * environ __attribute__((weak)); namespace nix { +void initLibUtil() { + initOpenSSL(); +} + std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 56160baaf..783a4a601 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,6 +32,9 @@ namespace nix { struct Sink; struct Source; +void initLibUtil(); + +void initOpenSSL(); /** * The system for which Nix is compiled. From a692c437298ad59004583f193ef3d73a378fd837 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 16:46:26 +0100 Subject: [PATCH 053/114] Move loadConfFile() to initLibStore Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. Using libstore without loading the config file is risky, as sqlite may then be misconfigured. See https://github.com/cachix/cachix/issues/475 --- perl/lib/Nix/Store.xs | 1 - src/libmain/shared.cc | 2 -- src/libstore/globals.cc | 3 +++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de91dc28d..b3f192810 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - loadConfFile(); settings.lockCPU = false; _store = openStore(); } catch (Error & e) { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2ed310cba..6dd64c6c7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -166,8 +166,6 @@ void initNix() if (sodium_init() == -1) throw Error("could not initialise libsodium"); - loadConfFile(); - startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..b18525dd7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -291,6 +291,9 @@ void assertLibStoreInitialized() { } void initLibStore() { + + loadConfFile(); + initLibStoreDone = true; } From 969307671500cb6cb9c01ab91c1d815ebd6a644b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:09:35 +0100 Subject: [PATCH 054/114] Move initLibStore() immediately after initLibUtil() Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. The goal of this reordering is to make initLibStore self-sufficient in a following commit. --- src/libmain/shared.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 6dd64c6c7..5e19bddb7 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -162,6 +162,7 @@ void initNix() #endif initLibUtil(); + initLibStore(); if (sodium_init() == -1) throw Error("could not initialise libsodium"); @@ -223,7 +224,6 @@ void initNix() #endif preloadNSS(); - initLibStore(); } From a58be394769fb174ee4b6ff5ce16744cf5806485 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:12:11 +0100 Subject: [PATCH 055/114] Move sodium_init() to initLibStore() Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 5 ----- src/libstore/globals.cc | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5e19bddb7..8e693fd8d 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -28,8 +28,6 @@ #include -#include - namespace nix { @@ -164,9 +162,6 @@ void initNix() initLibUtil(); initLibStore(); - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b18525dd7..1e66838c5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -13,6 +13,8 @@ #include +#include + namespace nix { @@ -292,6 +294,9 @@ void assertLibStoreInitialized() { void initLibStore() { + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + loadConfFile(); initLibStoreDone = true; From e706ffa007120249deace149dc4ba7cacf2c8beb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:24:14 +0100 Subject: [PATCH 056/114] Move preloadNSS() from initNix to initLibStore It is required for the sandbox, which is a libstore responsibility; not just libmain. Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 42 --------------------------------------- src/libstore/globals.cc | 44 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 8e693fd8d..cbd80756e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -20,11 +19,6 @@ #ifdef __linux__ #include #endif -#ifdef __GLIBC__ -#include -#include -#include -#endif #include @@ -113,41 +107,6 @@ std::string getArg(const std::string & opt, return *i; } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -218,7 +177,6 @@ void initNix() unsetenv("TMPDIR"); #endif - preloadNSS(); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1e66838c5..6848991a2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,11 @@ #include +#ifdef __GLIBC__ +#include +#include +#include +#endif namespace nix { @@ -283,6 +289,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -299,6 +341,8 @@ void initLibStore() { loadConfFile(); + preloadNSS(); + initLibStoreDone = true; } From 52d6ce6515ff1e8462b67b2adb1942477ce122f8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:35:28 +0100 Subject: [PATCH 057/114] Move macOS TMPDIR hack from initNix to initLibStore This code is bad. We shouldn't unset variables in programs whose children may need them. Fixing one issue at a time, so postponing. See https://github.com/NixOS/nix/issues/7731 Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 8 -------- src/libstore/globals.cc | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index cbd80756e..2a7e09e65 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -169,14 +169,6 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6848991a2..5a8825be5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -343,6 +343,14 @@ void initLibStore() { preloadNSS(); + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } From 1107ea363f600f37152e2b144d03c4071c2a6b6b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:41:24 +0100 Subject: [PATCH 058/114] libmain: Clarify the lack of initLibExpr() Quote Why not initLibExpr()? initGC() is essentially that, but detectStackOverflow is not an instance of the init function concept, as it may have to be invoked more than once per process. Furthermore, renaming initGC to initLibExpr is more trouble than it's worth at this time. --- src/libmain/shared.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2a7e09e65..a25865aad 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -156,7 +156,10 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for From 781d3dceb303d9fceabe9a39eae0f7f986e1adcc Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 17:43:14 +0100 Subject: [PATCH 059/114] Move initLibUtil() from initNix to initLibStore libutil is a dependency of libstore, so it should always be initialized as such. libutil is also a dependency of libmain. Being explicit about this dependency might be good, but not worth the slight code complexity until the library structure gets more advanced. Part of an effort to make it easier to initialize the right things, by moving code into the appropriate libraries. --- src/libmain/shared.cc | 1 - src/libstore/globals.cc | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a25865aad..56f47a4ac 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -118,7 +118,6 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - initLibUtil(); initLibStore(); startSignalHandlerThread(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5a8825be5..3f944f024 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -336,6 +336,8 @@ void assertLibStoreInitialized() { void initLibStore() { + initLibUtil(); + if (sodium_init() == -1) throw Error("could not initialise libsodium"); From 2196fd1146aa077419a113059ced924a648f9766 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 1 Feb 2023 18:38:54 +0100 Subject: [PATCH 060/114] libutil: Provide alternatives to startSignalHandlerThread How signals should be handled depends on what kind of process Nix is integrated into. The signal handler thread used by the stand-alone Nix commands / processes may not work well in the context of other runtime systems, such as those of Python, Perl, or Haskell. --- src/libutil/util.cc | 44 ++++++++++++++++++++++++++++++++++++++++++-- src/libutil/util.hh | 19 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 0099f7ebc..5c19dc737 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1748,13 +1748,39 @@ void triggerInterrupt() } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} void startSignalHandlerThread() { updateWindowSize(); - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); + saveSignalMask(); sigset_t set; sigemptyset(&set); @@ -1771,6 +1797,20 @@ void startSignalHandlerThread() static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 783a4a601..08993e1cf 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -448,6 +448,8 @@ void setStackSize(size_t stackSize); /** * Restore the original inherited Unix process context (such as signal * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). */ void restoreProcessContext(bool restoreMounts = true); @@ -817,9 +819,26 @@ class Callback; /** * Start a thread that handles various signals. Also block those signals * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; From 2445afd92c99ec0901a0e1a00fadda12aad15220 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 3 Feb 2023 18:07:47 +0100 Subject: [PATCH 061/114] Require openssl >= 1.1.1 Versions older this are sufficiently old that we don't want to support them, and they require extra support code. --- configure.ac | 2 +- src/libutil/hash.cc | 23 ----------------------- src/libutil/util.cc | 1 - src/libutil/util.hh | 2 -- 4 files changed, 1 insertion(+), 27 deletions(-) diff --git a/configure.ac b/configure.ac index f1f45f868..ba5756169 100644 --- a/configure.ac +++ b/configure.ac @@ -184,7 +184,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9df8bcfb4..02bddc8d9 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -17,29 +17,6 @@ namespace nix { -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -void initOpenSSL() { -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif -} - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5c19dc737..21d1c8dcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -48,7 +48,6 @@ extern char * * environ __attribute__((weak)); namespace nix { void initLibUtil() { - initOpenSSL(); } std::optional getEnv(const std::string & key) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 08993e1cf..6ff9d2524 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -34,8 +34,6 @@ struct Source; void initLibUtil(); -void initOpenSSL(); - /** * The system for which Nix is compiled. */ From 1c0b680ef9ca9604ff993a9d693355254ddc5bf4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 5 Feb 2023 14:16:27 +0100 Subject: [PATCH 062/114] libstore: Remove lockCPU dead code Left over from 9747ea84b, https://github.com/NixOS/nix/pull/5821 --- perl/lib/Nix/Store.xs | 1 - src/libstore/globals.cc | 1 - 2 files changed, 2 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index b3f192810..10a0c4067 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,7 +27,6 @@ static ref store() if (!_store) { try { initLibStore(); - settings.lockCPU = false; _store = openStore(); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 3f944f024..1b38e32fb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -49,7 +49,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); From ddebeb934a20225eec518520c96768bf00f0810a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 5 Feb 2023 14:16:27 +0100 Subject: [PATCH 063/114] libstore: Remove lockCPU dead code Left over from 9747ea84b, https://github.com/NixOS/nix/pull/5821 --- src/libstore/globals.hh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63c7389da..c29ad5f89 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -478,11 +478,6 @@ public: )", {"env-keep-derivations"}}; - /** - * Whether to lock the Nix client and worker to the same CPU. - */ - bool lockCPU; - Setting sandboxMode{ this, #if __linux__ From 73eb6a2a578c85fe4a9399bdc20b4ff943fb49b4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Apr 2023 11:01:23 -0400 Subject: [PATCH 064/114] Single page for experimental feature descriptions As requested by @fricklerhandwerk. --- .gitignore | 2 +- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/generate-xp-features.nix | 22 +++++-------------- doc/manual/local.mk | 6 ++--- doc/manual/src/SUMMARY.md.in | 1 - .../src/contributing/experimental-features.md | 4 ++++ 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index ffaf52be8..8ceff4ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,7 @@ perl/Makefile.config /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md -/doc/manual/src/contributing/experimental-features +/doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md # /scripts/ diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index b2095bc27..30e211c96 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ./utils.nix; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features/${name}.md) + - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index ff64edcf7..adb94355c 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,21 +1,11 @@ -xps: - with builtins; with import ./utils.nix; let - makePage = { name, value }: - { - name = "${name}.md"; - inherit value; - feature = name; - }; + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} - featurePages = map makePage (attrsToList xps); - - tableOfContents = let - showEntry = page: - " - [${page.feature}](contributing/experimental-features/${page.name})"; - in concatStringsSep "\n" (map showEntry featurePages) + "\n"; - -in (listToAttrs featurePages) // { "SUMMARY.md" = tableOfContents; } + ${doc} + ''; +in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index ae55ae4a9..63e7e61e4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -85,7 +85,7 @@ $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -107,7 +107,7 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-features: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -160,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-features $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 298644044..5bf274550 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -96,7 +96,6 @@ - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) - [Experimental Features](contributing/experimental-features.md) -{{#include ./contributing/experimental-features/SUMMARY.md}} - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md index f1db22751..ad5cffa91 100644 --- a/doc/manual/src/contributing/experimental-features.md +++ b/doc/manual/src/contributing/experimental-features.md @@ -89,3 +89,7 @@ However they serve different purposes: It is primarily an issue of *design* and *communication*, targeting the broader community. This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} From 4e0804c920558575a4b3486df1e595445bf67555 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 9 Apr 2023 22:42:20 +0200 Subject: [PATCH 065/114] Deduplicate string literal rendering, fix 4909 --- src/libcmd/repl.cc | 20 ++++++-------------- src/libexpr/eval.cc | 13 +++---------- src/libexpr/nixexpr.cc | 19 ++++--------------- src/libexpr/value/print.cc | 26 ++++++++++++++++++++++++++ src/libexpr/value/print.hh | 30 ++++++++++++++++++++++++++++++ src/libstore/derivations.cc | 9 +++++++++ tests/repl.sh | 8 ++++++++ 7 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 src/libexpr/value/print.cc create mode 100644 src/libexpr/value/print.hh diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 57848a5d3..1366622c7 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,6 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" +#include "value/print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -894,17 +895,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m } -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} // FIXME: lot of cut&paste from Nix's eval.cc. @@ -922,12 +912,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; + str << ANSI_CYAN; + printLiteral(str, v.boolean); + str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printStringValue(str, v.string.s); + printLiteral(str, v.string.s); str << ANSI_NORMAL; break; @@ -967,7 +959,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isVarName(i.first)) str << i.first; else - printStringValue(str, i.first.c_str()); + printLiteral(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 18cfd9531..06208897f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" +#include "value/print.hh" #include #include @@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - str << (boolean ? "true" : "false"); + printLiteral(str, boolean); break; case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; + printLiteral(str, string.s); break; case tPath: str << path; // !!! escaping? diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..ca6df0af3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" +#include "value/print.hh" #include @@ -62,18 +63,6 @@ Pos::operator std::shared_ptr() const /* Displaying abstract syntax trees. */ -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; @@ -85,7 +74,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) else { char c = s[0]; if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); + printLiteral(str, s); return str; } for (auto c : s) @@ -93,7 +82,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { - showString(str, s); + printLiteral(str, s); return str; } str << s; @@ -118,7 +107,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteral(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc new file mode 100644 index 000000000..cf97def46 --- /dev/null +++ b/src/libexpr/value/print.cc @@ -0,0 +1,26 @@ +#include "value/print.hh" + +namespace nix { + +std::ostream & +printLiteral(std::ostream & str, const std::string_view string) { + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + } + str << "\""; + return str; +} + +std::ostream & +printLiteral(std::ostream & str, bool boolean) { + str << (boolean ? "true" : "false"); + return str; +} + +} diff --git a/src/libexpr/value/print.hh b/src/libexpr/value/print.hh new file mode 100644 index 000000000..31c94eb85 --- /dev/null +++ b/src/libexpr/value/print.hh @@ -0,0 +1,30 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include + +namespace nix { + /** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param s The logical string + */ + std::ostream & printLiteral(std::ostream & o, std::string_view s); + inline std::ostream & printLiteral(std::ostream & o, const char * s) { + return printLiteral(o, std::string_view(s)); + } + inline std::ostream & printLiteral(std::ostream & o, const std::string & s) { + return printLiteral(o, std::string_view(s)); + } + + /** Print `true` or `false`. */ + std::ostream & printLiteral(std::ostream & o, bool b); +} diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index abdfb1978..9948862e5 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an std::string. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector buffer; diff --git a/tests/repl.sh b/tests/repl.sh index be8adb742..2b3789521 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -79,6 +79,14 @@ testReplResponse ' "result: ${a}" ' "result: 2" +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +testReplResponse ' +"$" + "{hi}" +' '"\\${hi}"' + testReplResponse ' drvPath ' '".*-simple.drv"' \ From deb7f4b466574c12a8384f1c76df0141c6f7ca69 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 11:29:35 +0200 Subject: [PATCH 066/114] Nitpicks --- src/libutil/config.hh | 4 ++-- src/libutil/experimental-features.cc | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 6baba0167..162626791 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -382,11 +382,11 @@ struct ExperimentalFeatureSettings : Config { experimental-features = nix-command flakes ``` - Experimental features available: + The following experimental features are available: {{#include experimental-features-shortlist.md}} - Experimental features are [further documented](@docroot@/contributing/experimental-features.md) in the contribution guide. + Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). )"}; /** diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1cd0dbb79..5b4418714 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -204,7 +204,7 @@ const std::optional parseExperimentalFeature(const std::str { using ReverseXpMap = std::map; - static std::unique_ptr reverseXpMap = [](){ + static std::unique_ptr reverseXpMap = []() { auto reverseXpMap = std::make_unique(); for (auto & xpFeature : xpFeatureDetails) (*reverseXpMap)[xpFeature.name] = xpFeature.tag; @@ -223,7 +223,8 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -nlohmann::json documentExperimentalFeatures() { +nlohmann::json documentExperimentalFeatures() +{ StringMap res; for (auto & xpFeature : xpFeatureDetails) res[std::string { xpFeature.name }] = From 7f5ca6192d091090bc71ab7bf96dd4acf0f1d376 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 10 Apr 2023 14:12:10 +0100 Subject: [PATCH 067/114] Add script to reproduce issue by inducing heavy load. --- repro-7998.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 repro-7998.sh diff --git a/repro-7998.sh b/repro-7998.sh new file mode 100755 index 000000000..b022d31ab --- /dev/null +++ b/repro-7998.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -eux +LOG_FILE=/tmp/repro-7998.log +rm -f "$LOG_FILE" +for i in {1..8}; do + ( + while true; do + nix-build \ + --argstr uuid $(uuidgen) \ + --arg drvCount $((RANDOM % 256)) \ + -E ' + { uuid ? "00000000-0000-0000-0000-000000000000", drvCount ? 0 }: + with import { }; + let + mkDrv = name: buildInputs: + stdenv.mkDerivation { + inherit name; + inherit buildInputs; + unpackPhase = "date +\"${uuid} %F %T\" >date.txt"; + installPhase = "mkdir -p $out; cp date.txt $out/"; + }; + mkDrvs = n: + let + name = "repro-7998-${toString n}"; + buildInputs = if n == 0 then [ ] else [ (mkDrvs (n - 1)) ]; + in mkDrv name buildInputs; + in mkDrvs drvCount + ' + done 2>&1 | tee -a "$LOG_FILE" + ) & +done +read # Press enter to stop +pkill -KILL -f repro-7998.sh From 7c56e842133afe14812270c34cda3dc0a3da8aa6 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 10:22:07 +0100 Subject: [PATCH 068/114] Warn after a second of being busy instead of immediately. Getting the occasional SQLITE_BUSY is expected when the database is being accessed concurrently. The retry will likely succeed so it is pointless to warn immediately. Instead we track how long each retrySQLite block has been running, and only begin warning after a second has elapsed (and then every 10 seconds subsequently). --- src/libstore/sqlite.cc | 9 ++------- src/libstore/sqlite.hh | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 871f2f3be..c57e58fe0 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,14 +239,9 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn) { - static std::atomic lastWarned{0}; - - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (shouldWarn) { logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index b735838ec..e2c9e28f8 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -139,7 +139,7 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn); /** * Convenience function for retrying a SQLite transaction when the @@ -148,11 +148,22 @@ void handleSQLiteBusy(const SQLiteBusy & e); template T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); + } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + time_t now = time(0); + bool shouldWarn = false; + + if (now > nextWarning) { + nextWarning = now + 10; + shouldWarn = true; + } + + handleSQLiteBusy(e, shouldWarn); } } } From da322ebda62470acbf5373374c4cee8236705c2f Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 10:47:53 +0100 Subject: [PATCH 069/114] Revert "Add script to reproduce issue by inducing heavy load." This reverts commit 213b838f9cfb820d2bc76d7c6edc468b27029945. --- repro-7998.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100755 repro-7998.sh diff --git a/repro-7998.sh b/repro-7998.sh deleted file mode 100755 index b022d31ab..000000000 --- a/repro-7998.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -eux -LOG_FILE=/tmp/repro-7998.log -rm -f "$LOG_FILE" -for i in {1..8}; do - ( - while true; do - nix-build \ - --argstr uuid $(uuidgen) \ - --arg drvCount $((RANDOM % 256)) \ - -E ' - { uuid ? "00000000-0000-0000-0000-000000000000", drvCount ? 0 }: - with import { }; - let - mkDrv = name: buildInputs: - stdenv.mkDerivation { - inherit name; - inherit buildInputs; - unpackPhase = "date +\"${uuid} %F %T\" >date.txt"; - installPhase = "mkdir -p $out; cp date.txt $out/"; - }; - mkDrvs = n: - let - name = "repro-7998-${toString n}"; - buildInputs = if n == 0 then [ ] else [ (mkDrvs (n - 1)) ]; - in mkDrv name buildInputs; - in mkDrvs drvCount - ' - done 2>&1 | tee -a "$LOG_FILE" - ) & -done -read # Press enter to stop -pkill -KILL -f repro-7998.sh From e570a91661f0742f07dd148e4e75e943d7806f2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 12:40:14 +0200 Subject: [PATCH 070/114] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.15.md | 58 +++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 56 ------------------------ 3 files changed, 59 insertions(+), 56 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.15.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 5bf274550..f783d5908 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -99,6 +99,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md new file mode 100644 index 000000000..133121999 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-` and `man nix-env-` + * `nix-store --help --` and `nix-env --help --`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 5b62836bf..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,58 +1,2 @@ # Release X.Y (202?-??-??) -* Commands which take installables on the command line can now read them from the standard input if - passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which - exceed the OS arg limit. - -* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` - or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` - or `--to-sri` to convert a hash to Base64 or SRI format, respectively. - - As the choice of hash formats is no longer binary, the `--base16` flag is also added - to explicitly specify the Base16 format, which is still the default. - -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. - - The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. - Using this is better and more clear than relying on the now-removed `.drv` special handling. - - For example, - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv - ``` - - now gives info about the derivation itself, while - - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* - ``` - provides information about each of its outputs. - -* The experimental command `nix describe-stores` has been removed. - -* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - -* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. - They include all common options that can be specified and common environment variables that affect these commands. - - These pages can be viewed offline with `man` using - - * `man nix-store-` and `man nix-env-` - * `nix-store --help --` and `nix-env --help --`. - -* Nix when used as a client now checks whether the store (the server) trusts the client. - (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) - This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. - - `nix store ping` and `nix doctor` now display this information. - -* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language. - It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. - It uses the same JSON layout as `nix show-derivation`, and is its inverse. - -* `nix show-derivation` has been renamed to `nix derivation show`. - This matches `nix derivation add`, and avoids bloating the top-level namespace. - The old name is still kept as an alias for compatibility, however. - -* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. - This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. From 450e5ec6185e2e1102e67ec7a348a0dc8955692d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 11 Apr 2023 10:46:38 -0400 Subject: [PATCH 071/114] Do not gate or hide experimental settings This is somewhat hacky fix just for 2.15. I unintentionally hid them from the manual, when no one wanted to hide them that (including myself). I also required the experimental feature to be enabled in an order-dependent way, which is not good. The simplest fix for this immanent release is just to always show them, and always allow them to be set. Effectively undoes some changes from aa663b7e89d3d02248d37ee9f68b52770b247018 --- src/libutil/config.cc | 16 +++---------- tests/experimental-features.sh | 42 ++++++++++++++++++---------------- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8d63536d6..5ff8d91ba 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. -static bool applicable(const Config::SettingData & sd) -{ - return !sd.isAlias - && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); -} - void Config::getSettings(std::map & res, bool overriddenOnly) { for (auto & opt : _settings) - if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -154,7 +147,7 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (applicable(s.second)) + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; } @@ -163,7 +156,7 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (applicable(s.second)) + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; } @@ -171,9 +164,6 @@ std::string Config::toKeyValue() void Config::convertToArgs(Args & args, const std::string & category) { for (auto & s : _settings) { - /* We do include args for settings gated on disabled - experimental-features. The args themselves however will also be - gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); } diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index a4d55f5f4..73554da8c 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -1,25 +1,27 @@ source common.sh -# Without flakes, flake options should not show up -# With flakes, flake options should show up - -function both_ways { - nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake - nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake - - # Also, the order should not matter - nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake - nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake -} - -# Simple case, the configuration effects the running command -both_ways show-config - -# Skipping for now, because we actually *do* want these to show up in -# the manual, just be marked experimental. Will reenable once the manual -# generation takes advantage of the JSON metadata on this. - -# both_ways store gc --help +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' From de3df3009bf003f327d35e246d5904d93273e2e9 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Tue, 11 Apr 2023 16:03:37 +0100 Subject: [PATCH 072/114] Move warning timing logic into handleSQLiteBusy. --- src/libstore/sqlite.cc | 6 ++++-- src/libstore/sqlite.hh | 13 ++----------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index c57e58fe0..df334c23c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,9 +239,11 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - if (shouldWarn) { + time_t now = time(0); + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index e2c9e28f8..6e14852cb 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -139,7 +139,7 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e, bool shouldWarn); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); /** * Convenience function for retrying a SQLite transaction when the @@ -153,17 +153,8 @@ T retrySQLite(F && fun) while (true) { try { return fun(); - } catch (SQLiteBusy & e) { - time_t now = time(0); - bool shouldWarn = false; - - if (now > nextWarning) { - nextWarning = now + 10; - shouldWarn = true; - } - - handleSQLiteBusy(e, shouldWarn); + handleSQLiteBusy(e, nextWarning); } } } From a4b6d1d9a3a3234258790dd58c7041ee8fe01929 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Apr 2023 20:16:37 +0200 Subject: [PATCH 073/114] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index c910885a0..752490696 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.15.0 \ No newline at end of file +2.16.0 From ee97f107e8749d61ae5e310a2d457637ccc109ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 13 Apr 2023 13:38:12 -0400 Subject: [PATCH 074/114] Push `getFSAccessor` `unsupported(...)` down `Store` class hierarchy More progress on issue #5729. Instead of having it by the default method in `Store` itself, have it be the implementation in `DummyStore` and `LegacySSHStore`. Then just the implementations which fail to provide the method pay the "penalty" of dealing with the icky `unimplemented` function for non-compliance. Combined with my other recent PRs, this finally makes `Store` have no `unsupported` calls! --- src/libstore/dummy-store.cc | 3 +++ src/libstore/legacy-ssh-store.cc | 3 +++ src/libstore/store-api.hh | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index ae2777d0c..74d6ed3b5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -71,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override { callback(nullptr); } + + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eb471d8fc..d2ddbbe5f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -342,6 +342,9 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref getFSAccessor() override + { unsupported("getFSAccessor"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3cb48bff5..5bee272bf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -678,8 +678,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() - { unsupported("getFSAccessor"); } + virtual ref getFSAccessor() = 0; /** * Repair the contents of the given path by redownloading it using From 9e8f2090365a509656dead69bc91fb6615cf9d05 Mon Sep 17 00:00:00 2001 From: Raphael Robatsch Date: Thu, 13 Apr 2023 16:22:45 +0200 Subject: [PATCH 075/114] Display valid installable in InstallableDerivedPath::parse warning The warning message should produce an installable name that can be passed to `nix build`, `nix path-info`, etc. again. Since the CLI expects that the .drv path and the output names are separated by a caret, the warning message must also separate the .drv path and output names with a caret. However, `DerivedPath::Built.to_string()` uses an exclamation point as the separator instead. This commit adds a `separator` argument to the to_string method. This changes the warning message from: If this command is now failing try again with '/nix/store/foo.drv!*' to: If this command is now failing try again with '/nix/store/foo.drv^*' --- src/libcmd/installable-derived-path.cc | 2 +- src/libstore/derived-path.cc | 13 +++++++------ src/libstore/derived-path.hh | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..35f6c5bfe 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -47,7 +47,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( }; warn( "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", - oldDerivedPath.to_string(*store)); + oldDerivedPath.to_string(*store, '^')); }; return DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e5f0f1b33..6baca70a3 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -59,18 +59,19 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const Store & store, char separator) const { return store.printStorePath(drvPath) - + "!" + + separator + outputs.to_string(); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const Store & store, char separator) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store, separator); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 2155776b1..9fc64e2f2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -48,7 +48,7 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; - std::string to_string(const Store & store) const; + std::string to_string(const Store & store, char separator = '!') const; static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref store) const; @@ -81,7 +81,7 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } - std::string to_string(const Store & store) const; + std::string to_string(const Store & store, char separator = '!') const; static DerivedPath parse(const Store & store, std::string_view); }; From d93e76fbb80c9f407ddac128357350f54add68fb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Apr 2023 11:09:01 -0400 Subject: [PATCH 076/114] Start cross-referencing experimental features - Create a glossary entry for experimental features. - Have the man page experimental feature notice link `nix-commmand`. (Eventually this should be programmed, based on whether the command is experimental, and if so what experimental feature does it depend on.) - Document which installables depend on which experimental features. I tried to use the same style (bold warning and block quote) that the top of the man page uses. Co-authored-by: Valentin Gagarin --- doc/manual/generate-manpage.nix | 4 ++- .../src/command-ref/experimental-commands.md | 2 +- doc/manual/src/glossary.md | 6 +++++ .../src/language/advanced-attributes.md | 26 +++++++++++++++---- src/nix/nix.md | 14 +++++++++- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 86f2ca567..0b3a23801 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -10,7 +10,9 @@ let result = '' > **Warning** \ - > This program is **experimental** and its interface is subject to change. + > This program is + > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. # Name diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index cfa6f8b73..286ddc6d6 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 4eedb2e93..a9782be5c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -225,3 +225,9 @@ [string]: ./language/values.md#type-string [path]: ./language/values.md#type-path [attribute name]: ./language/values.md#attribute-set + + - [experimental feature]{#gloss-experimental-feature}\ + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3e8c48890..307971434 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes. about converting to and from base-32 notation.) - [`__contentAddressed`]{#adv-attr-__contentAddressed} - If this **experimental** attribute is set to true, then the derivation + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivations` experimental feature is enabled. - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather @@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes. - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ > **Warning** - > This is an experimental feature. + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). > - > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > To use this attribute, you must enable the + > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` > extra-experimental-features = discard-references diff --git a/src/nix/nix.md b/src/nix/nix.md index e1865b31c..1ef6c7fcd 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/). # Installables +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. + Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. The following types of installable are supported by most commands: -- [Flake output attribute](#flake-output-attribute) +- [Flake output attribute](#flake-output-attribute) (experimental) - [Store path](#store-path) - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path @@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a From bfc558c972aa8d6f5ef15a3e720bed964925ae32 Mon Sep 17 00:00:00 2001 From: Archit Gupta Date: Fri, 14 Apr 2023 11:33:38 -0700 Subject: [PATCH 077/114] Whitelist commit-lockfile-summary in flake nixConfig --- src/libexpr/flake/config.cc | 2 +- src/nix/flake.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 89ddbde7e..e89014862 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList) void ConfigFile::apply() { - std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"}; + std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lockfile-summary"}; for (auto & [name, value] : settings) { diff --git a/src/nix/flake.md b/src/nix/flake.md index d70f34eeb..965f6eb48 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -382,9 +382,9 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without - confirmation so long as `accept-flake-config` is not set in the global - configuration. + `bash-prompt-suffix`, `flake-registry`, and `commit-lockfile-summary`) + are allowed to be set without confirmation so long as `accept-flake-config` + is not set in the global configuration. ## Flake inputs From ee420ac64e7d1f51f5abcb069dbe84cd6ff707ce Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Apr 2023 20:45:11 -0400 Subject: [PATCH 078/114] Legacy vs non-legacy `to_string`/`parse` for `DerivedPath` As requested by @roberth, it is good to call out the specific instances we care about, which is `!` for the RPC protocols, and `^` for humans. This doesn't take advantage of parametricity as much, but since the human and computer interfaces are good to decouple anyways (we don't care if they drift further apart over time in the slightest) some separation and slight duplication is fine. Also, unit test both round trips. --- src/libcmd/installable-derived-path.cc | 2 +- src/libstore/derived-path.cc | 37 +++++++++++++++++++++----- src/libstore/derived-path.hh | 30 ++++++++++++++++++--- src/libstore/remote-store.cc | 4 +-- src/libstore/tests/derived-path.cc | 8 ++++++ 5 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 35f6c5bfe..6ecf54b7c 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -47,7 +47,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( }; warn( "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", - oldDerivedPath.to_string(*store, '^')); + oldDerivedPath.to_string(*store)); }; return DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 6baca70a3..9a2ffda39 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -59,17 +59,32 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store, char separator) const +std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) - + separator + + '^' + outputs.to_string(); } -std::string DerivedPath::to_string(const Store & store, char separator) const +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ + return store.printStorePath(drvPath) + + '!' + + outputs.to_string(); +} + +std::string DerivedPath::to_string(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store, separator); }, + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -88,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9fc64e2f2..5f7acbebc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -48,8 +48,18 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; - std::string to_string(const Store & store, char separator = '!') const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref store) const; GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); @@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } - std::string to_string(const Store & store, char separator = '!') const; + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e128c3a29..b862902d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) DerivedPath read(const Store & store, Source & from, Phantom _) { auto s = readString(from); - return DerivedPath::parse(store, s); + return DerivedPath::parseLegacy(store, s); } void write(const Store & store, Sink & out, const DerivedPath & req) { - out << req.to_string(store); + out << req.to_string_legacy(store); } diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d1ac2c5e7..e6d32dbd0 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init) { } +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_round_rip, From 9df7f3f5379ba79e6b40fb73bb91604cc7116c85 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 3 Mar 2021 01:11:09 +0000 Subject: [PATCH 079/114] Introduce `Worker::makeGoal` This takes a `DerivedPath` so the caller doesn't need to care about which sort of goal does what. --- src/libstore/build/entry-points.cc | 24 ++++-------------------- src/libstore/build/worker.cc | 15 +++++++++++++++ src/libstore/build/worker.hh | 12 ++++++++++-- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 2925fe3ca..7d725b881 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -10,16 +10,8 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); @@ -55,16 +47,8 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) { - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); - }, - [&](const DerivedPath::Opaque & bo) { - goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); - }, - }, br.raw()); - } + for (const auto & br : reqs) + goals.insert(worker.makeGoal(br, buildMode)); worker.run(goals); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f775f8486..6ad4a0e2b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -92,6 +92,7 @@ std::shared_ptr Worker::makePathSubstitutionGoal(const Sto return goal; } + std::shared_ptr Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional ca) { std::weak_ptr & goal_weak = drvOutputSubstitutionGoals[id]; @@ -104,6 +105,20 @@ std::shared_ptr Worker::makeDrvOutputSubstitutionGoal return goal; } + +GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) -> GoalPtr { + return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + }, + [&](const DerivedPath::Opaque & bo) -> GoalPtr { + return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); + }, + }, req.raw()); +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 48a1a27fa..bb51d641d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -181,7 +181,7 @@ public: */ /** - * derivation goal + * @ref DerivationGoal "derivation goal" */ private: std::shared_ptr makeDerivationGoalCommon( @@ -196,11 +196,19 @@ public: const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); /** - * substitution goal + * @ref SubstitutionGoal "substitution goal" */ std::shared_ptr makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); std::shared_ptr makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + /** + * Make a goal corresponding to the `DerivedPath`. + * + * It will be a `DerivationGoal` for a `DerivedPath::Built` or + * a `SubstitutionGoal` for a `DerivedPath::Opaque`. + */ + GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal); + /** * Remove a dead goal. */ From 37fca662b0acef3c104a159709a394832e297dda Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Mar 2022 01:26:07 +0000 Subject: [PATCH 080/114] Make `KeyedBuildResult`, `BuildResult` like before, and fix bug another way In https://github.com/NixOS/nix/pull/6311#discussion_r834863823, I realized since derivation goals' wanted outputs can "grow" due to overlapping dependencies (See `DerivationGoal::addWantedOutputs`, called by `Worker::makeDerivationGoalCommon`), the previous bug fix had an unfortunate side effect of causing more pointless rebuilds. In paticular, we have this situation: 1. Goal made from `DerivedPath::Built { foo, {a} }`. 2. Goal gives on on substituting, starts building. 3. Goal made from `DerivedPath::Built { foo, {b} }`, in fact is just modified original goal. 4. Though the goal had gotten as far as building, so all outputs were going to be produced, `addWantedOutputs` no longer knows that and so the goal is flagged to be restarted. This might sound far-fetched with input-addressed drvs, where we usually basically have all our goals "planned out" before we start doing anything, but with CA derivation goals and especially RFC 92, where *drv resolution* means goals are created after some building is completed, it is more likely to happen. So the first thing to do was restore the clearing of `wantedOutputs` we used to do, and then filter the outputs in `buildPathsWithResults` to only get the ones we care about. But fix also has its own side effect in that the `DerivedPath` in the `BuildResult` in `DerivationGoal` cannot be trusted; it is merely the *first* `DerivedPath` for which this goal was originally created. To remedy this, I made `BuildResult` be like it was before, and instead made `KeyedBuildResult` be a subclass wit the path. Only `buildPathsWithResults` returns `KeyedBuildResult`s, everything else just becomes like it was before, where the "key" is unambiguous from context. I think separating the "primary key" field(s) from the other fields is good practical in general anyways. (I would like to do the same thing for `ValidPathInfo`.) Among other things, it allows constructions like `std::map` where doesn't contain duplicate keys and just precludes the possibility of those duplicate keys being out of sync. We might leverage the above someday to overload `buildPathsWithResults` to take a *set* of return a *map* per the above. ----- Unfortunately, we need to avoid C++20 strictness on designated initializers. (BTW https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2287r1.html this offers some new syntax for this use-case. Hopefully this will be adopted and we can eventually use it.) No having that yet, maybe it would be better to not make `KeyedBuildResult` a subclass to just avoid this. Co-authored-by: Robert Hensing --- src/libstore/build-result.hh | 16 ++++--- src/libstore/build/derivation-goal.cc | 30 ++++++++++--- src/libstore/build/entry-points.cc | 29 +++++++----- src/libstore/build/goal.cc | 23 ++++++++++ src/libstore/build/goal.hh | 16 ++++++- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/legacy-ssh-store.cc | 9 +--- src/libstore/remote-store.cc | 50 +++++++++++++-------- src/libstore/remote-store.hh | 2 +- src/libstore/store-api.hh | 3 +- src/libstore/worker-protocol.hh | 1 + 11 files changed, 130 insertions(+), 51 deletions(-) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 27d1a1b6c..e07296eab 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -83,11 +83,6 @@ struct BuildResult */ bool isNonDeterministic = false; - /** - * The derivation we built or the store path we substituted. - */ - DerivedPath path; - /** * For derivations, a mapping from the names of the wanted outputs * to actual paths. @@ -116,4 +111,15 @@ struct BuildResult } }; +/** + * A `BuildResult` together with its "primary key". + */ +struct KeyedBuildResult : BuildResult +{ + /** + * The derivation we built or the store path we substituted. + */ + DerivedPath path; +}; + } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 26faf8c8e..606f9fd48 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -330,6 +330,10 @@ void DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = OutputsSpec::All { }; + /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); if (useDerivation) @@ -570,8 +574,6 @@ void DerivationGoal::inputsRealised() build hook. */ state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - - buildResult = BuildResult { .path = buildResult.path }; } void DerivationGoal::started() @@ -1452,12 +1454,28 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (waitee->buildResult.success()) - if (auto bfd = std::get_if(&waitee->buildResult.path)) - for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + if (!useDerivation) return; + auto & fullDrv = *dynamic_cast(drv.get()); + + auto * dg = dynamic_cast(&*waitee); + if (!dg) return; + + auto outputs = fullDrv.inputDrvs.find(dg->drvPath); + if (outputs == fullDrv.inputDrvs.end()) return; + + for (auto & outputName : outputs->second) { + auto buildResult = dg->getBuildResult(DerivedPath::Built { + .drvPath = dg->drvPath, + .outputs = OutputsSpec::Names { outputName }, + }); + if (buildResult.success()) { + for (auto & [output, realisation] : buildResult.builtOutputs) { inputDrvOutputs.insert_or_assign( - { bfd->drvPath, output.outputName }, + { dg->drvPath, output.outputName }, realisation.outPath); + } + } + } } } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 7d725b881..74eae0692 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -39,7 +39,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } } -std::vector Store::buildPathsWithResults( +std::vector Store::buildPathsWithResults( const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) @@ -47,15 +47,23 @@ std::vector Store::buildPathsWithResults( Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; - for (const auto & br : reqs) - goals.insert(worker.makeGoal(br, buildMode)); + std::vector> state; + + for (const auto & req : reqs) { + auto goal = worker.makeGoal(req, buildMode); + goals.insert(goal); + state.push_back({req, goal}); + } worker.run(goals); - std::vector results; + std::vector results; - for (auto & i : goals) - results.push_back(i->buildResult); + for (auto & [req, goalPtr] : state) + results.emplace_back(KeyedBuildResult { + goalPtr->getBuildResult(req), + /* .path = */ req, + }); return results; } @@ -68,15 +76,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); - return goal->buildResult; + return goal->getBuildResult(DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All {}, + }); } catch (Error & e) { return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, }; }; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index d59b94797..13b2e509a 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,6 +11,29 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } +BuildResult Goal::getBuildResult(const DerivedPath & req) { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first.outputName)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + void addToWeakGoals(WeakGoals & goals, GoalPtr p) { if (goals.find(p) != goals.end()) diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index f4bf6f38b..c0e12a2ed 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -81,11 +81,26 @@ struct Goal : public std::enable_shared_from_this */ ExitCode exitCode = ecBusy; +protected: /** * Build result. */ BuildResult buildResult; +public: + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &); + /** * Exception containing an error message, if any. */ @@ -93,7 +108,6 @@ struct Goal : public std::enable_shared_from_this Goal(Worker & worker, DerivedPath path) : worker(worker) - , buildResult { .path = std::move(path) } { } virtual ~Goal() diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 58d6901d3..af937f6b1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1335,7 +1335,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo result.rethrow(); } - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr) override diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d2ddbbe5f..7b40b27e0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -287,12 +287,7 @@ public: conn->to.flush(); - BuildResult status { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult status; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; @@ -330,7 +325,7 @@ public: conn->to.flush(); - BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; + BuildResult result; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b862902d1..734e6f27f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -125,10 +125,26 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput) } -BuildResult read(const Store & store, Source & from, Phantom _) +KeyedBuildResult read(const Store & store, Source & from, Phantom _) { auto path = worker_proto::read(store, from, Phantom {}); - BuildResult res { .path = path }; + auto br = worker_proto::read(store, from, Phantom {}); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void write(const Store & store, Sink & to, const KeyedBuildResult & res) +{ + worker_proto::write(store, to, res.path); + worker_proto::write(store, to, static_cast(res)); +} + + +BuildResult read(const Store & store, Source & from, Phantom _) +{ + BuildResult res; res.status = (BuildResult::Status) readInt(from); from >> res.errorMsg @@ -142,7 +158,6 @@ BuildResult read(const Store & store, Source & from, Phantom _) void write(const Store & store, Sink & to, const BuildResult & res) { - worker_proto::write(store, to, res.path); to << res.status << res.errorMsg @@ -865,7 +880,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod readInt(conn->from); } -std::vector RemoteStore::buildPathsWithResults( +std::vector RemoteStore::buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) @@ -880,7 +895,7 @@ std::vector RemoteStore::buildPathsWithResults( writeDerivedPaths(*this, conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom> {}); + return worker_proto::read(*this, conn->from, Phantom> {}); } else { // Avoid deadlock. conn_.reset(); @@ -889,21 +904,25 @@ std::vector RemoteStore::buildPathsWithResults( // fails, but meh. buildPaths(paths, buildMode, evalStore); - std::vector results; + std::vector results; for (auto & path : paths) { std::visit( overloaded { [&](const DerivedPath::Opaque & bo) { - results.push_back(BuildResult { - .status = BuildResult::Substituted, - .path = bo, + results.push_back(KeyedBuildResult { + { + .status = BuildResult::Substituted, + }, + /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - BuildResult res { - .status = BuildResult::Built, - .path = bfd, + KeyedBuildResult res { + { + .status = BuildResult::Built + }, + /* .path = */ bfd, }; OutputPathMap outputs; @@ -952,12 +971,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = OutputsSpec::All { }, - }, - }; + BuildResult res; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1c45f543e..a30466647 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -114,7 +114,7 @@ public: void buildPaths(const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; - std::vector buildPathsWithResults( + std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5bee272bf..9d6880de9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -92,6 +92,7 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; +struct KeyedBuildResult; typedef std::map> StorePathCAMap; @@ -575,7 +576,7 @@ public: * case of a build/substitution error, this function won't throw an * exception, but return a BuildResult containing an error message. */ - virtual std::vector buildPathsWithResults( + virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c7a6f8688..34b2fc17b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, KeyedBuildResult); MAKE_WORKER_PROTO(, std::optional); MAKE_WORKER_PROTO(template, std::vector); From 0f2b5146c79895ac10362b6da56b535fc3d963a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 14 Feb 2023 13:25:55 -0500 Subject: [PATCH 081/114] Make restarting state machines explicit If my memory is correct, @edolstra objected to modifying `wantedOutputs` upon falling back to doing a build (as we did before), because we should only modify it in response to new requests --- *actual* wants --- and not because we are "incidentally" building all the outptus beyond what may have been requested. That's a fair point, and the alternative is to replace the boolean soup with proper enums: Instead of modifying `wantedOuputs` som more, we'll modify `needsRestart` to indicate we are passed the need. --- src/libstore/build/derivation-goal.cc | 49 +++++++++++++++++++------ src/libstore/build/derivation-goal.hh | 52 ++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 606f9fd48..5bb664bff 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -145,8 +145,20 @@ void DerivationGoal::work() void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { auto newWanted = wantedOutputs.union_(outputs); - if (!newWanted.isSubsetOf(wantedOutputs)) - needRestart = true; + switch (needRestart) { + case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: + if (!newWanted.isSubsetOf(wantedOutputs)) + needRestart = NeedRestartForMoreOutputs::OutputsAddedDoNeed; + break; + case NeedRestartForMoreOutputs::OutputsAddedDoNeed: + /* No need to check whether we added more outputs, because a + restart is already queued up. */ + break; + case NeedRestartForMoreOutputs::BuildInProgressWillNotNeed: + /* We are already building all outputs, so it doesn't matter if + we now want more. */ + break; + }; wantedOutputs = newWanted; } @@ -297,12 +309,29 @@ void DerivationGoal::outputsSubstitutionTried() In particular, it may be the case that the hole in the closure is an output of the current derivation, which causes a loop if retried. */ - if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; + { + bool substitutionFailed = + nrIncompleteClosure > 0 && + nrIncompleteClosure == nrFailed; + switch (retrySubstitution) { + case RetrySubstitution::NoNeed: + if (substitutionFailed) + retrySubstitution = RetrySubstitution::YesNeed; + break; + case RetrySubstitution::YesNeed: + // Should not be able to reach this state from here. + assert(false); + break; + case RetrySubstitution::AlreadyRetried: + debug("substitution failed again, but we already retried once. Not retrying again."); + break; + } + } nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - if (needRestart) { - needRestart = false; + if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { + needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; haveDerivation(); return; } @@ -330,9 +359,9 @@ void DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs = OutputsSpec::All { }; + /* At this point we are building all outputs, so if more are wanted there + is no need to restart. */ + needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); @@ -455,8 +484,8 @@ void DerivationGoal::inputsRealised() return; } - if (retrySubstitution && !retriedSubstitution) { - retriedSubstitution = true; + if (retrySubstitution == RetrySubstitution::YesNeed) { + retrySubstitution = RetrySubstitution::AlreadyRetried; haveDerivation(); return; } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 3a6f0c2d9..9b5bd1805 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -78,22 +78,58 @@ struct DerivationGoal : public Goal */ std::map, StorePath> inputDrvOutputs; + /** + * See `needRestart`; just for that field. + */ + enum struct NeedRestartForMoreOutputs { + /** + * The goal state machine is progressing based on the current value of + * `wantedOutputs. No actions are needed. + */ + OutputsUnmodifedDontNeed, + /** + * `wantedOutputs` has been extended, but the state machine is + * proceeding according to its old value, so we need to restart. + */ + OutputsAddedDoNeed, + /** + * The goal state machine has progressed to the point of doing a build, + * in which case all outputs will be produced, so extensions to + * `wantedOutputs` no longer require a restart. + */ + BuildInProgressWillNotNeed, + }; + /** * Whether additional wanted outputs have been added. */ - bool needRestart = false; + NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + + /** + * See `retrySubstitution`; just for that field. + */ + enum RetrySubstitution { + /** + * No issues have yet arose, no need to restart. + */ + NoNeed, + /** + * Something failed and there is an incomplete closure. Let's retry + * substituting. + */ + YesNeed, + /** + * We are current or have already retried substitution, and whether or + * not something goes wrong we will not retry again. + */ + AlreadyRetried, + }; /** * Whether to retry substituting the outputs after building the * inputs. This is done in case of an incomplete closure. */ - bool retrySubstitution = false; - - /** - * Whether we've retried substitution, in which case we won't try - * again. - */ - bool retriedSubstitution = false; + RetrySubstitution retrySubstitution = RetrySubstitution::NoNeed; /** * The derivation stored at drvPath. From 24866b71c40f0fcb5a601d90d4f87366fe626090 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 14 Apr 2023 18:18:32 -0400 Subject: [PATCH 082/114] Introduce `SingleDrvOutputs` In many cases we are dealing with a collection of realisations, they are all outputs of the same derivation. In that case, we don't need "derivation hashes modulos" to be part of our map key, because the output names alone will be unique. Those hashes are still part of the realisation proper, so we aren't loosing any information, we're just "normalizing our schema" by narrowing the "primary key". Besides making our data model a bit "tighter" this allows us to avoid a double `for` loop in `DerivationGoal::waiteeDone`. The inner `for` loop was previously just to select the output we cared about without knowing its hash. Now we can just select the output by name directly. Note that neither protocol is changed as part of this: we are still transferring `DrvOutputs` over the wire for `BuildResult`s. I would only consider revising this once #6223 is merged, and we can mention protocol versions inside factored-out serialization logic. Until then it is better not change anything because it would come a the cost of code reuse. --- src/build-remote/build-remote.cc | 5 ++-- src/libcmd/installables.cc | 4 +-- src/libstore/build-result.hh | 2 +- src/libstore/build/derivation-goal.cc | 26 ++++++++-------- src/libstore/build/derivation-goal.hh | 12 ++++---- src/libstore/build/goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 ++-- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/daemon.cc | 5 +++- src/libstore/legacy-ssh-store.cc | 6 +++- src/libstore/realisation.hh | 33 ++++++++++++++++++++- src/libstore/remote-store.cc | 20 +++++++++---- src/nix-store/nix-store.cc | 5 +++- 13 files changed, 90 insertions(+), 38 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index cfc4baaca..ce9c7f45a 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -311,8 +311,9 @@ connected: auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; if (!store->queryRealisation(thisOutputId)) { debug("missing output %s", outputName); - assert(result.builtOutputs.count(thisOutputId)); - auto newRealisation = result.builtOutputs.at(thisOutputId); + auto i = result.builtOutputs.find(outputName); + assert(i != result.builtOutputs.end()); + auto & newRealisation = i->second; missingRealisations.insert(newRealisation); missingPaths.insert(newRealisation.outPath); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 32ae46d9f..0a2fe0073 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -593,8 +593,8 @@ std::vector, BuiltPathWithResult>> Installable::build std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); + for (auto & [outputName, realisation] : buildResult.builtOutputs) + outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { .path = BuiltPath::Built { bfd.drvPath, outputs }, .info = aux.info, diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index e07296eab..b7a56e791 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -87,7 +87,7 @@ struct BuildResult * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; /** * The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5bb664bff..a4bb94b0e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1013,7 +1013,7 @@ void DerivationGoal::resolvedFinished() auto resolvedDrv = *resolvedDrvGoal->drv; auto & resolvedResult = resolvedDrvGoal->buildResult; - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; if (resolvedResult.success()) { auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); @@ -1039,7 +1039,7 @@ void DerivationGoal::resolvedFinished() worker.store.printStorePath(drvPath), wantedOutput); auto realisation = [&]{ - auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + auto take1 = get(resolvedResult.builtOutputs, wantedOutput); if (take1) return *take1; /* The above `get` should work. But sateful tracking of @@ -1064,7 +1064,7 @@ void DerivationGoal::resolvedFinished() worker.store.registerDrvOutput(newRealisation); } outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + builtOutputs.emplace(wantedOutput, realisation); } runPostBuildHook( @@ -1189,7 +1189,7 @@ HookReply DerivationGoal::tryBuildHook() } -DrvOutputs DerivationGoal::registerOutputs() +SingleDrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1351,7 +1351,7 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() } -std::pair DerivationGoal::checkPathValidity() +std::pair DerivationGoal::checkPathValidity() { if (!drv->type().isPure()) return { false, {} }; @@ -1364,7 +1364,7 @@ std::pair DerivationGoal::checkPathValidity() return static_cast(names); }, }, wantedOutputs.raw()); - DrvOutputs validOutputs; + SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { auto initialOutput = get(initialOutputs, i.first); @@ -1407,7 +1407,7 @@ std::pair DerivationGoal::checkPathValidity() } } if (info.wanted && info.known && info.known->isValid()) - validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); + validOutputs.emplace(i.first, Realisation { drvOutput, info.known->path }); } // If we requested all the outputs, we are always fine. @@ -1431,7 +1431,7 @@ std::pair DerivationGoal::checkPathValidity() } -DrvOutputs DerivationGoal::assertPathValidity() +SingleDrvOutputs DerivationGoal::assertPathValidity() { auto [allValid, validOutputs] = checkPathValidity(); if (!allValid) @@ -1442,7 +1442,7 @@ DrvOutputs DerivationGoal::assertPathValidity() void DerivationGoal::done( BuildResult::Status status, - DrvOutputs builtOutputs, + SingleDrvOutputs builtOutputs, std::optional ex) { buildResult.status = status; @@ -1498,11 +1498,11 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { - for (auto & [output, realisation] : buildResult.builtOutputs) { + auto i = buildResult.builtOutputs.find(outputName); + if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg->drvPath, output.outputName }, - realisation.outPath); - } + { dg->drvPath, outputName }, + i->second.outPath); } } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 9b5bd1805..7033b7a58 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -253,7 +253,7 @@ struct DerivationGoal : public Goal * Check that the derivation outputs all exist and register them * as valid. */ - virtual DrvOutputs registerOutputs(); + virtual SingleDrvOutputs registerOutputs(); /** * Open a log file and a pipe to it. @@ -306,17 +306,17 @@ struct DerivationGoal : public Goal * Update 'initialOutputs' to determine the current status of the * outputs of the derivation. Also returns a Boolean denoting * whether all outputs are valid and non-corrupt, and a - * 'DrvOutputs' structure containing the valid and wanted + * 'SingleDrvOutputs' structure containing the valid and wanted * outputs. */ - std::pair checkPathValidity(); + std::pair checkPathValidity(); /** * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'DrvOutputs' structure containing the wanted + * returns a 'SingleDrvOutputs' structure containing the wanted * outputs. */ - DrvOutputs assertPathValidity(); + SingleDrvOutputs assertPathValidity(); /** * Forcibly kill the child process, if any. @@ -329,7 +329,7 @@ struct DerivationGoal : public Goal void done( BuildResult::Status status, - DrvOutputs builtOutputs = {}, + SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); void waiteeDone(GoalPtr waitee, ExitCode result) override; diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 13b2e509a..ca7097a68 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -23,7 +23,7 @@ BuildResult Goal::getBuildResult(const DerivedPath & req) { */ for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { - if (bp.outputs.contains(it->first.outputName)) + if (bp.outputs.contains(it->first)) ++it; else it = res.builtOutputs.erase(it); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index af937f6b1..6cb483a9c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2174,7 +2174,7 @@ void LocalDerivationGoal::runChild() } -DrvOutputs LocalDerivationGoal::registerOutputs() +SingleDrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2691,7 +2691,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - DrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { auto oldinfo = get(initialOutputs, outputName); @@ -2710,7 +2710,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() worker.store.registerDrvOutput(thisRealisation); } if (wantedOutputs.contains(outputName)) - builtOutputs.emplace(thisRealisation.id, thisRealisation); + builtOutputs.emplace(outputName, thisRealisation); } return builtOutputs; diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 42d32a31a..9acd7593d 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -237,7 +237,7 @@ struct LocalDerivationGoal : public DerivationGoal * Check that the derivation outputs all exist and register them * as valid. */ - DrvOutputs registerOutputs() override; + SingleDrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 63898f8dc..621a59c0a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -637,7 +637,10 @@ static void performOp(TunnelLogger * logger, ref store, to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; } if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - worker_proto::write(*store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, to, builtOutputs); } break; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7b40b27e0..6e50fe6cd 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -294,7 +294,11 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - status.builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return status; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index a18cf2aa8..3922d1267 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -13,9 +13,25 @@ namespace nix { class Store; +/** + * A general `Realisation` key. + * + * This is similar to a `DerivedPath::Opaque`, but the derivation is + * identified by its "hash modulo" instead of by its store path. + */ struct DrvOutput { - // The hash modulo of the derivation + /** + * The hash modulo of the derivation. + * + * Computed from the derivation itself for most types of + * derivations, but computed from the (fixed) content address of the + * output for fixed-output derivations. + */ Hash drvHash; + + /** + * The name of the output. + */ std::string outputName; std::string to_string() const; @@ -60,6 +76,21 @@ struct Realisation { GENERATE_CMP(Realisation, me->id, me->outPath); }; +/** + * Collection type for a single derivation's outputs' `Realisation`s. + * + * Since these are the outputs of a single derivation, we know the + * output names are unique so we can use them as the map key. + */ +typedef std::map SingleDrvOutputs; + +/** + * Collection type for multiple derivations' outputs' `Realisation`s. + * + * `DrvOutput` is used because in general the derivations are not all + * the same, so we need to identify firstly which derivation, and + * secondly which output of that derivation. + */ typedef std::map DrvOutputs; struct OpaquePath { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 734e6f27f..69e809a0f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -152,7 +152,11 @@ BuildResult read(const Store & store, Source & from, Phantom _) >> res.isNonDeterministic >> res.startTime >> res.stopTime; - res.builtOutputs = worker_proto::read(store, from, Phantom {}); + auto builtOutputs = worker_proto::read(store, from, Phantom {}); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); return res; } @@ -165,7 +169,10 @@ void write(const Store & store, Sink & to, const BuildResult & res) << res.isNonDeterministic << res.startTime << res.stopTime; - worker_proto::write(store, to, res.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(store, to, builtOutputs); } @@ -941,10 +948,10 @@ std::vector RemoteStore::buildPathsWithResults( queryRealisation(outputId); if (!realisation) throw MissingRealisation(outputId); - res.builtOutputs.emplace(realisation->id, *realisation); + res.builtOutputs.emplace(output, *realisation); } else { res.builtOutputs.emplace( - outputId, + output, Realisation { .id = outputId, .outPath = outputPath, @@ -979,7 +986,10 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { auto builtOutputs = worker_proto::read(*this, conn->from, Phantom {}); - res.builtOutputs = builtOutputs; + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); } return res; } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7035e6a7b..74f255bee 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -935,7 +935,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 3) out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - worker_proto::write(*store, out, status.builtOutputs); + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + worker_proto::write(*store, out, builtOutputs); } break; From 9c74df5bb4f41c938a4f6942492f5ce92b0ef371 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Apr 2023 20:56:51 +0200 Subject: [PATCH 083/114] Format Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson --- src/libexpr/value/print.cc | 6 ++++-- src/libstore/derivations.cc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc index cf97def46..c67ff9f87 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/value/print.cc @@ -3,7 +3,8 @@ namespace nix { std::ostream & -printLiteral(std::ostream & str, const std::string_view string) { +printLiteral(std::ostream & str, const std::string_view string) +{ str << "\""; for (auto i = string.begin(); i != string.end(); ++i) { if (*i == '\"' || *i == '\\') str << "\\" << *i; @@ -18,7 +19,8 @@ printLiteral(std::ostream & str, const std::string_view string) { } std::ostream & -printLiteral(std::ostream & str, bool boolean) { +printLiteral(std::ostream & str, bool boolean) +{ str << (boolean ? "true" : "false"); return str; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9948862e5..7eb5cd275 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -314,7 +314,7 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi /** - * Print a derivation string literal to an std::string. + * Print a derivation string literal to an `std::string`. * * This syntax does not generalize to the expression language, which needs to * escape `$`. From 1e2dd669bcdd8df6cdaac061e035828626906447 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 12:56:31 +0200 Subject: [PATCH 084/114] printLiteral: Do not overload --- src/libcmd/repl.cc | 6 +++--- src/libexpr/eval.cc | 4 ++-- src/libexpr/nixexpr.cc | 6 +++--- src/libexpr/value/print.cc | 4 ++-- src/libexpr/value/print.hh | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1366622c7..41cf77424 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -913,13 +913,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nBool: str << ANSI_CYAN; - printLiteral(str, v.boolean); + printLiteralBool(str, v.boolean); str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printLiteral(str, v.string.s); + printLiteralString(str, v.string.s); str << ANSI_NORMAL; break; @@ -959,7 +959,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isVarName(i.first)) str << i.first; else - printLiteral(str, i.first); + printLiteralString(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 06208897f..bd05fc156 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -105,10 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - printLiteral(str, boolean); + printLiteralBool(str, boolean); break; case tString: - printLiteral(str, string.s); + printLiteralString(str, string.s); break; case tPath: str << path; // !!! escaping? diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index ca6df0af3..1b5d522d3 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -74,7 +74,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) else { char c = s[0]; if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - printLiteral(str, s); + printLiteralString(str, s); return str; } for (auto c : s) @@ -82,7 +82,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { - printLiteral(str, s); + printLiteralString(str, s); return str; } str << s; @@ -107,7 +107,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - printLiteral(str, s); + printLiteralString(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/value/print.cc b/src/libexpr/value/print.cc index c67ff9f87..bd241d9d8 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/value/print.cc @@ -3,7 +3,7 @@ namespace nix { std::ostream & -printLiteral(std::ostream & str, const std::string_view string) +printLiteralString(std::ostream & str, const std::string_view string) { str << "\""; for (auto i = string.begin(); i != string.end(); ++i) { @@ -19,7 +19,7 @@ printLiteral(std::ostream & str, const std::string_view string) } std::ostream & -printLiteral(std::ostream & str, bool boolean) +printLiteralBool(std::ostream & str, bool boolean) { str << (boolean ? "true" : "false"); return str; diff --git a/src/libexpr/value/print.hh b/src/libexpr/value/print.hh index 31c94eb85..98dd2008d 100644 --- a/src/libexpr/value/print.hh +++ b/src/libexpr/value/print.hh @@ -17,14 +17,14 @@ namespace nix { * * @param s The logical string */ - std::ostream & printLiteral(std::ostream & o, std::string_view s); - inline std::ostream & printLiteral(std::ostream & o, const char * s) { - return printLiteral(o, std::string_view(s)); + std::ostream & printLiteralString(std::ostream & o, std::string_view s); + inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); } - inline std::ostream & printLiteral(std::ostream & o, const std::string & s) { - return printLiteral(o, std::string_view(s)); + inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); } /** Print `true` or `false`. */ - std::ostream & printLiteral(std::ostream & o, bool b); + std::ostream & printLiteralBool(std::ostream & o, bool b); } From 28a5cdde02964306e7eb443f696c8d5d59ebf9e9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 13:10:45 +0200 Subject: [PATCH 085/114] libexpr/value/print.* -> libexpr/print.* Generalizes the file to sensibly allow printing any part of the language syntax. --- src/libcmd/repl.cc | 2 +- src/libexpr/eval.cc | 2 +- src/libexpr/nixexpr.cc | 2 +- src/libexpr/{value => }/print.cc | 2 +- src/libexpr/{value => }/print.hh | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename src/libexpr/{value => }/print.cc (96%) rename src/libexpr/{value => }/print.hh (100%) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 41cf77424..806dce024 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,7 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" -#include "value/print.hh" +#include "print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bd05fc156..6668add8c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,7 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" -#include "value/print.hh" +#include "print.hh" #include #include diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 1b5d522d3..d8f3cd701 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,7 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" -#include "value/print.hh" +#include "print.hh" #include diff --git a/src/libexpr/value/print.cc b/src/libexpr/print.cc similarity index 96% rename from src/libexpr/value/print.cc rename to src/libexpr/print.cc index bd241d9d8..282903b72 100644 --- a/src/libexpr/value/print.cc +++ b/src/libexpr/print.cc @@ -1,4 +1,4 @@ -#include "value/print.hh" +#include "print.hh" namespace nix { diff --git a/src/libexpr/value/print.hh b/src/libexpr/print.hh similarity index 100% rename from src/libexpr/value/print.hh rename to src/libexpr/print.hh From b6125772d7d5f82d48873fc93a7f261832154b14 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Apr 2023 14:07:35 +0200 Subject: [PATCH 086/114] libexpr: Move identifier-like printing to print.cc --- src/libcmd/repl.cc | 6 ++--- src/libexpr/nixexpr.cc | 27 +++-------------------- src/libexpr/print.cc | 50 ++++++++++++++++++++++++++++++++++++++++++ src/libexpr/print.hh | 18 +++++++++++++++ 4 files changed, 73 insertions(+), 28 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 806dce024..80c08bf1c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -426,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) } +// FIXME: DRY and match or use the parser static bool isVarName(std::string_view s) { if (s.size() == 0) return false; @@ -956,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m sorted.emplace(state->symbols[i.name], i.value); for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printLiteralString(str, i.first); + printAttributeName(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d8f3cd701..1557cbbeb 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -61,33 +61,12 @@ Pos::operator std::shared_ptr() const return pos; } -/* Displaying abstract syntax trees. */ - +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; - - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - printLiteralString(str, s); - return str; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - printLiteralString(str, s); - return str; - } - str << s; - } - return str; + return printIdentifier(str, s); } void Expr::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 282903b72..d08672cfc 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -25,4 +25,54 @@ printLiteralBool(std::ostream & str, bool boolean) return str; } +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +// FIXME: keywords +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + + } diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh index 98dd2008d..f9cfc3964 100644 --- a/src/libexpr/print.hh +++ b/src/libexpr/print.hh @@ -27,4 +27,22 @@ namespace nix { /** Print `true` or `false`. */ std::ostream & printLiteralBool(std::ostream & o, bool b); + + /** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ + std::ostream & printAttributeName(std::ostream & o, std::string_view s); + + /** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ + std::ostream & printIdentifier(std::ostream & o, std::string_view s); } From ba9ae691b6b0d8f6e23a2a9468aef51a75f16cfe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 16 Apr 2023 10:44:03 -0400 Subject: [PATCH 087/114] Add `optionalString` to manual Nix lang utilities Use it everywhere it could be also. --- doc/manual/generate-manpage.nix | 28 +++++++++++++++------------- doc/manual/utils.nix | 4 +++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 0b3a23801..d04eecf55 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -31,19 +31,18 @@ let showSynopsis = command: args: let - showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "..."); + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' `${command}` [*option*...] ${arguments} ''; - maybeSubcommands = if details ? commands && details.commands != {} - then '' + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' where *subcommand* is one of the following: ${subcommands} - '' - else ""; + ''; subcommands = if length categories > 1 then listCategories @@ -65,12 +64,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = - if details ? doc - then replaceStrings ["@stores@"] [storeDocs] details.doc - else ""; + maybeDocumentation = optionalString + (details ? doc) + (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = if details.flags == {} then "" else '' + maybeOptions = optionalString (details.flags != {}) '' # Options ${showOptions details.flags toplevel.flags} @@ -80,15 +78,19 @@ let let allOptions = options // commonOptions; showCategory = cat: '' - ${if cat != "" then "**${cat}:**" else ""} + ${optionalString (cat != "") "**${cat}:**"} ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} ''; listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let - shortName = if option ? shortName then "/ `-${option.shortName}`" else ""; - labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else ""; + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - `--${name}` ${shortName} ${labels} diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 82544935a..e69cbe658 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -42,6 +42,8 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); + optionalString = cond: string: if cond then string else ""; + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: let result = squash '' @@ -74,7 +76,7 @@ rec { else "*machine-specific*"; showAliases = aliases: - if aliases == [] then "" else + optionalString (aliases != []) "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; in result; From 9800c1e8074d248f75ea9bed1b5a0f76e799863d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 10 Apr 2023 12:02:35 -0400 Subject: [PATCH 088/114] Mark experimental configuration settings programmatically Fix #8162 The test is changed to compare `nlohmann::json` values, not strings of dumped JSON, which allows us to format things more nicely. --- doc/manual/utils.nix | 20 +++++++++++++++- src/libstore/globals.hh | 20 ---------------- src/libutil/config.cc | 4 ++++ src/libutil/tests/config.cc | 48 ++++++++++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index e69cbe658..9043dd8cd 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -44,7 +44,7 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - ${if useAnchors @@ -54,10 +54,28 @@ rec { ${indent " " body} ''; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${name} = ... + ``` + ''; + # separate body to cleanly handle indentation body = '' ${description} + ${experimentalFeatureNote} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 63c7389da..f598ed4a8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -328,16 +328,6 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` )"}; Setting startId{this, @@ -367,16 +357,6 @@ public: Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 5ff8d91ba..a42f3a849 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -191,6 +191,10 @@ std::map AbstractSetting::toJSONObject() std::map obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 8be6730dd..f250e934e 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -156,12 +156,54 @@ namespace nix { } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map settings; - Setting setting{&config, "", "name-of-the-setting", "description"}; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) { From d0cf615cbbd1f4b15e04cb3696af109066096aef Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Apr 2023 14:27:26 +0200 Subject: [PATCH 089/114] add link to `nix-conf` setting Co-authored-by: John Ericson --- doc/manual/src/command-ref/opt-common.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index c94b6aef8..ad4e99e21 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,7 +203,7 @@ Most Nix commands accept the following command-line options: instead. - [`-I`](#opt-I) *path*\ - Add a path to the Nix expression search path. + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). This option may be given multiple times. Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). From f4119a67ccb12ab8988ccb5d0fd19660a20d61d0 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Apr 2023 14:54:30 +0200 Subject: [PATCH 090/114] use @docroot@ link --- doc/manual/src/command-ref/opt-common.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index ad4e99e21..7a012250d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -205,7 +205,7 @@ Most Nix commands accept the following command-line options: - [`-I`](#opt-I) *path*\ Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](./env-common.md#env-NIX_PATH). + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value*\ Set the Nix configuration option *name* to *value*. This overrides From 537e8719f2ca8e18312bd8dcc37124fb1b25d4d3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 09:15:11 -0400 Subject: [PATCH 091/114] Explain various `.self = false,` Co-authored-by: Robert Hensing --- src/libstore/binary-cache-store.cc | 2 ++ src/libstore/local-store.cc | 1 + src/nix/profile.cc | 1 + 3 files changed, 4 insertions(+) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 628e9b9db..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, @@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore( }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0072a16dc..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1328,6 +1328,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 5aa87a313..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -209,6 +209,7 @@ struct ProfileManifest }, .references = { .others = std::move(references), + // profiles never refer to themselves .self = false, }, }, From 2c8475600d16e463a9c63aa76aee9f6152128f14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 9 Apr 2023 22:39:04 -0400 Subject: [PATCH 092/114] Fix some issues with experimental config settings Issues: 1. Features gated on disabled experimental settings should warn and be ignored, not silently succeed. 2. Experimental settings in the same config "batch" (file or env var) as the enabling of the experimental feature should work. 3. For (2), the order should not matter. These are analogous to the issues @roberth caught with my changes for arg handling, but they are instead for config handling. Co-authored-by: Robert Hensing --- src/libstore/globals.cc | 27 +++++----- src/libstore/globals.hh | 4 +- src/libutil/config-impl.hh | 71 +++++++++++++++++++++++++ src/libutil/config.cc | 94 ++++++++++++++++++++-------------- src/libutil/config.hh | 46 +++++++++++++++-- src/libutil/tests/config.cc | 2 + tests/experimental-features.sh | 60 +++++++++++++++++++--- 7 files changed, 238 insertions(+), 66 deletions(-) create mode 100644 src/libutil/config-impl.hh diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1b38e32fb..4c66d08ee 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -22,6 +22,9 @@ #include #endif +#include "config-impl.hh" + + namespace nix { @@ -192,18 +195,18 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str, bool append) +template<> SandboxMode BaseSetting::parse(const std::string & str) const { - if (str == "true") value = smEnabled; - else if (str == "relaxed") value = smRelaxed; - else if (str == "false") value = smDisabled; + if (str == "true") return smEnabled; + else if (str == "relaxed") return smRelaxed; + else if (str == "false") return smDisabled; else throw UsageError("option '%s' has invalid value '%s'", name, str); } -template<> bool BaseSetting::isAppendable() +template<> struct BaseSetting::trait { - return false; -} + static constexpr bool appendable = false; +}; template<> std::string BaseSetting::to_string() const { @@ -235,23 +238,23 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str, bool append) +unsigned int MaxBuildJobsSetting::parse(const std::string & str) const { - if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + if (str == "auto") return std::max(1U, std::thread::hardware_concurrency()); else { if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); } } -void PluginFilesSetting::set(const std::string & str, bool append) +Paths PluginFilesSetting::parse(const std::string & str) const { if (pluginsLoaded) throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - BaseSetting::set(str, append); + return BaseSetting::parse(str); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d6c5d437a..609cf53b8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -26,7 +26,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + unsigned int parse(const std::string & str) const override; }; struct PluginFilesSetting : public BaseSetting @@ -43,7 +43,7 @@ struct PluginFilesSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Paths parse(const std::string & str) const override; }; const uint32_t maxIdsPerBuild = diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh new file mode 100644 index 000000000..b6cae5ec3 --- /dev/null +++ b/src/libutil/config-impl.hh @@ -0,0 +1,71 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * One only needs to include this when one is declaring a + * `BaseClass` setting, or as derived class of such an + * instantiation. + */ + +#include "config.hh" + +namespace nix { + +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting::trait +{ + static constexpr bool appendable = true; +}; +template<> struct BaseSetting>::trait +{ + static constexpr bool appendable = true; +}; + +template +struct BaseSetting::trait +{ + static constexpr bool appendable = false; +}; + +template +bool BaseSetting::isAppendable() +{ + return trait::appendable; +} + +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); + +template +void BaseSetting::appendOrSet(T && newValue, bool append) +{ + static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + assert(!append); + value = std::move(newValue); +} + +template +void BaseSetting::set(const std::string & str, bool append) +{ + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) + appendOrSet(parse(str), append); + else { + assert(experimentalFeature); + warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", + name, + showExperimentalFeature(*experimentalFeature)); + } +} + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index a42f3a849..085a884dc 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -3,6 +3,8 @@ #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "config-impl.hh" + #include namespace nix { @@ -80,6 +82,8 @@ void Config::getSettings(std::map & res, bool overridd void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { unsigned int pos = 0; + std::vector> parsedContents; + while (pos < contents.size()) { std::string line; while (pos < contents.size() && contents[pos] != '\n') @@ -125,8 +129,21 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string auto i = tokens.begin(); advance(i, 2); - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + parsedContents.push_back({ + name, + concatStringsSep(" ", Strings(i, tokens.end())), + }); }; + + // First apply experimental-feature related settings + for (auto & [name, value] : parsedContents) + if (name == "experimental-features" || name == "extra-experimental-features") + set(name, value); + + // Then apply other settings + for (auto & [name, value] : parsedContents) + if (name != "experimental-features" && name != "extra-experimental-features") + set(name, value); } void AbstractConfig::applyConfigFile(const Path & path) @@ -202,12 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -bool BaseSetting::isAppendable() -{ - return false; -} - template void BaseSetting::convertToArg(Args & args, const std::string & category) { @@ -231,9 +242,9 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> std::string BaseSetting::parse(const std::string & str) const { - value = str; + return str; } template<> std::string BaseSetting::to_string() const @@ -242,11 +253,11 @@ template<> std::string BaseSetting::to_string() const } template -void BaseSetting::set(const std::string & str, bool append) +T BaseSetting::parse(const std::string & str) const { static_assert(std::is_integral::value, "Integer required."); if (auto n = string2Int(str)) - value = *n; + return *n; else throw UsageError("setting '%s' has invalid value '%s'", name, str); } @@ -258,12 +269,12 @@ std::string BaseSetting::to_string() const return std::to_string(value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> bool BaseSetting::parse(const std::string & str) const { if (str == "true" || str == "yes" || str == "1") - value = true; + return true; else if (str == "false" || str == "no" || str == "0") - value = false; + return false; else throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); } @@ -291,16 +302,15 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> Strings BaseSetting::parse(const std::string & str) const { - auto ss = tokenizeString(str); - if (!append) value.clear(); - for (auto & s : ss) value.push_back(std::move(s)); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) value.push_back(std::move(s)); } template<> std::string BaseSetting::to_string() const @@ -308,16 +318,16 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringSet BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); - for (auto & s : tokenizeString(str)) - value.insert(s); + return tokenizeString(str); } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting::to_string() const @@ -325,21 +335,24 @@ template<> std::string BaseSetting::to_string() const return concatStringsSep(" ", value); } -template<> void BaseSetting>::set(const std::string & str, bool append) +template<> std::set BaseSetting>::parse(const std::string & str) const { - if (!append) value.clear(); + std::set res; for (auto & s : tokenizeString(str)) { auto thisXpFeature = parseExperimentalFeature(s); if (thisXpFeature) - value.insert(thisXpFeature.value()); + res.insert(thisXpFeature.value()); else warn("unknown experimental feature '%s'", s); } + return res; } -template<> bool BaseSetting>::isAppendable() +template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && s : std::move(newValue)) + value.insert(s); } template<> std::string BaseSetting>::to_string() const @@ -350,20 +363,23 @@ template<> std::string BaseSetting>::to_string() c return concatStringsSep(" ", stringifiedXpFeatures); } -template<> void BaseSetting::set(const std::string & str, bool append) +template<> StringMap BaseSetting::parse(const std::string & str) const { - if (!append) value.clear(); + StringMap res; for (auto & s : tokenizeString(str)) { auto eq = s.find_first_of('='); if (std::string::npos != eq) - value.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); + res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } + return res; } -template<> bool BaseSetting::isAppendable() +template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) { - return true; + if (!append) value.clear(); + for (auto && [k, v] : std::move(newValue)) + value.emplace(std::move(k), std::move(v)); } template<> std::string BaseSetting::to_string() const @@ -387,15 +403,15 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; -void PathSetting::set(const std::string & str, bool append) +Path PathSetting::parse(const std::string & str) const { if (str == "") { if (allowEmpty) - value = ""; + return ""; else throw UsageError("setting '%s' cannot be empty", name); } else - value = canonPath(str); + return canonPath(str); } bool GlobalConfig::set(const std::string & name, const std::string & value) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 162626791..2675baed7 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -215,8 +215,11 @@ protected: virtual void set(const std::string & value, bool append = false) = 0; - virtual bool isAppendable() - { return false; } + /** + * Whether the type is appendable; i.e. whether the `append` + * parameter to `set()` is allowed to be `true`. + */ + virtual bool isAppendable() = 0; virtual std::string to_string() const = 0; @@ -241,6 +244,23 @@ protected: const T defaultValue; const bool documentDefault; + /** + * Parse the string into a `T`. + * + * Used by `set()`. + */ + virtual T parse(const std::string & str) const; + + /** + * Append or overwrite `value` with `newValue`. + * + * Some types to do not support appending in which case `append` + * should never be passed. The default handles this case. + * + * @param append Whether to append or overwrite. + */ + virtual void appendOrSet(T && newValue, bool append); + public: BaseSetting(const T & def, @@ -268,9 +288,25 @@ public: template void setDefault(const U & v) { if (!overridden) value = v; } - void set(const std::string & str, bool append = false) override; + /** + * Require any experimental feature the setting depends on + * + * Uses `parse()` to get the value from `str`, and `appendOrSet()` + * to set it. + */ + void set(const std::string & str, bool append = false) override final; - bool isAppendable() override; + /** + * C++ trick; This is template-specialized to compile-time indicate whether + * the type is appendable. + */ + struct trait; + + /** + * Always defined based on the C++ magic + * with `trait` above. + */ + bool isAppendable() override final; virtual void override(const T & v) { @@ -336,7 +372,7 @@ public: options->addSetting(this); } - void set(const std::string & str, bool append = false) override; + Path parse(const std::string & str) const override; Path operator +(const char * p) const { return value + p; } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index f250e934e..886e70da5 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -82,6 +82,7 @@ namespace nix { TestSetting() : AbstractSetting("test", "test", {}) {} void set(const std::string & value, bool append) override {} std::string to_string() const override { return {}; } + bool isAppendable() override { return false; } }; Config config; @@ -90,6 +91,7 @@ namespace nix { ASSERT_FALSE(config.set("test", "value")); config.addSetting(&setting); ASSERT_TRUE(config.set("test", "value")); + ASSERT_FALSE(config.set("extra-test", "value")); } TEST(Config, withInitialValue) { diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index 73554da8c..607bf0a8e 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -23,20 +23,64 @@ source common.sh # # Medium case, the configuration effects --help # grep_both_ways store gc --help -expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' -nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' +# Test settings that are gated on experimental features; the setting is ignored +# with a warning if the experimental feature is not enabled. The order of the +# `setting = value` lines in the configuration should not matter. + +# 'flakes' experimental-feature is disabled before, ignore and warn +NIX_CONFIG=' + experimental-features = nix-command + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is disabled after, ignore and warn +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "false" $TEST_ROOT/stdout +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled before, process +NIX_CONFIG=' + experimental-features = nix-command flakes + accept-flake-config = true +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +# 'flakes' experimental-feature is enabled after, process +NIX_CONFIG=' + accept-flake-config = true + experimental-features = nix-command flakes +' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +grepQuiet "true" $TEST_ROOT/stdout +grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr + +function exit_code_both_ways { + expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null + nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + + # Also, the order should not matter + expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null + nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null +} + +exit_code_both_ways show-config --flake-registry 'https://no' # Double check these are stable -nix --experimental-features '' --help -nix --experimental-features '' doctor --help -nix --experimental-features '' repl --help -nix --experimental-features '' upgrade-nix --help +nix --experimental-features '' --help 1>/dev/null +nix --experimental-features '' doctor --help 1>/dev/null +nix --experimental-features '' repl --help 1>/dev/null +nix --experimental-features '' upgrade-nix --help 1>/dev/null # These 3 arguments are currently given to all commands, which is wrong (as not # all care). To deal with fixing later, we simply make them require the # nix-command experimental features --- it so happens that the commands we wish # stabilizing to do not need them anyways. for arg in '--print-build-logs' '--offline' '--refresh'; do - nix --experimental-features 'nix-command' "$arg" --help - ! nix --experimental-features '' "$arg" --help + nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null + expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null done From d41e1bed5e1e1f87927ca1e8e6e1c1ad18b1ea7f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 09:41:39 -0400 Subject: [PATCH 093/114] Experimentally allow forcing `nix-daemon` trust; use this to test We finally test the status quo of remote build trust in a number of ways. We create a new experimental feature on `nix-daemon` to do so. PR #3921, which improves the situation with trustless remote building, will build upon these changes. This code / tests was pull out of there to make this, so everything is easier to review, and in particular we test before and after so the new behavior in that PR is readily apparent from the testsuite diff alone. --- src/libstore/daemon.cc | 2 + src/libutil/experimental-features.cc | 12 +++- src/libutil/experimental-features.hh | 1 + src/nix/daemon.cc | 63 ++++++++++++++----- tests/build-remote-trustless-after.sh | 2 + tests/build-remote-trustless-should-fail-0.sh | 29 +++++++++ tests/build-remote-trustless-should-pass-0.sh | 9 +++ tests/build-remote-trustless-should-pass-1.sh | 9 +++ tests/build-remote-trustless-should-pass-3.sh | 14 +++++ tests/build-remote-trustless.sh | 14 +++++ tests/local.mk | 4 ++ tests/nix-daemon-untrusting.sh | 3 + 12 files changed, 145 insertions(+), 17 deletions(-) create mode 100644 tests/build-remote-trustless-after.sh create mode 100644 tests/build-remote-trustless-should-fail-0.sh create mode 100644 tests/build-remote-trustless-should-pass-0.sh create mode 100644 tests/build-remote-trustless-should-pass-1.sh create mode 100644 tests/build-remote-trustless-should-pass-3.sh create mode 100644 tests/build-remote-trustless.sh create mode 100755 tests/nix-daemon-untrusting.sh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0d7ec2af0..af9a76f1e 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1067,6 +1067,8 @@ void processConnection( opCount++; + debug("performing daemon worker op: %d", op); + try { performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); } catch (Error & e) { diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 5b4418714..bd1899662 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -189,6 +189,16 @@ constexpr std::array xpFeatureDetails = {{ runtime dependencies. )", }, + { + .tag = Xp::DaemonTrustOverride, + .name = "daemon-trust-override", + .description = R"( + Allow forcing trusting or not trusting clients with + `nix-daemon`. This is useful for testing, but possibly also + useful for various experiments with `nix-daemon --stdio` + networking. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 8ef66263a..3c00bc4e5 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -28,6 +28,7 @@ enum struct ExperimentalFeature AutoAllocateUids, Cgroups, DiscardReferences, + DaemonTrustOverride, }; /** diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7ae7b4ea6..c1a91c63d 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -273,8 +273,12 @@ static std::pair authPeer(const PeerInfo & peer) /** * Run a server. The loop opens a socket and accepts new connections from that * socket. + * + * @param forceTrustClientOpt If present, force trusting or not trusted + * the client. Otherwise, decide based on the authentication settings + * and user credentials (from the unix domain socket). */ -static void daemonLoop() +static void daemonLoop(std::optional forceTrustClientOpt) { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -317,9 +321,18 @@ static void daemonLoop() closeOnExec(remote.get()); - PeerInfo peer = getPeerInfo(remote.get()); - auto [_trusted, user] = authPeer(peer); - auto trusted = _trusted; + PeerInfo peer { .pidKnown = false }; + TrustedFlag trusted; + std::string user; + + if (forceTrustClientOpt) + trusted = *forceTrustClientOpt; + else { + peer = getPeerInfo(remote.get()); + auto [_trusted, _user] = authPeer(peer); + trusted = _trusted; + user = _user; + }; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "", @@ -410,38 +423,47 @@ static void forwardStdioConnection(RemoteStore & store) { * Unlike `forwardStdioConnection()` we do process commands ourselves in * this case, not delegating to another daemon. * - * @note `Trusted` is unconditionally passed because in this mode we - * blindly trust the standard streams. Limiting access to those is - * explicitly not `nix-daemon`'s responsibility. + * @param trustClient Whether to trust the client. Forwarded directly to + * `processConnection()`. */ -static void processStdioConnection(ref store) +static void processStdioConnection(ref store, TrustedFlag trustClient) { FdSource from(STDIN_FILENO); FdSink to(STDOUT_FILENO); - processConnection(store, from, to, Trusted, NotRecursive); + processConnection(store, from, to, trustClient, NotRecursive); } /** * Entry point shared between the new CLI `nix daemon` and old CLI * `nix-daemon`. + * + * @param forceTrustClientOpt See `daemonLoop()` and the parameter with + * the same name over there for details. */ -static void runDaemon(bool stdio) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt) { if (stdio) { auto store = openUncachedStore(); - if (auto remoteStore = store.dynamic_pointer_cast()) + // If --force-untrusted is passed, we cannot forward the connection and + // must process it ourselves (before delegating to the next store) to + // force untrusting the client. + if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) forwardStdioConnection(*remoteStore); else - processStdioConnection(store); + // `Trusted` is passed in the auto (no override case) because we + // cannot see who is on the other side of a plain pipe. Limiting + // access to those is explicitly not `nix-daemon`'s responsibility. + processStdioConnection(store, forceTrustClientOpt.value_or(Trusted)); } else - daemonLoop(); + daemonLoop(forceTrustClientOpt); } static int main_nix_daemon(int argc, char * * argv) { { auto stdio = false; + std::optional isTrustedOpt = std::nullopt; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -452,11 +474,20 @@ static int main_nix_daemon(int argc, char * * argv) printVersion("nix-daemon"); else if (*arg == "--stdio") stdio = true; - else return false; + else if (*arg == "--force-trusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = Trusted; + } else if (*arg == "--force-untrusted") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = NotTrusted; + } else if (*arg == "--default-trust") { + experimentalFeatureSettings.require(Xp::DaemonTrustOverride); + isTrustedOpt = std::nullopt; + } else return false; return true; }); - runDaemon(stdio); + runDaemon(stdio, isTrustedOpt); return 0; } @@ -482,7 +513,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false); + runDaemon(false, std::nullopt); } }; diff --git a/tests/build-remote-trustless-after.sh b/tests/build-remote-trustless-after.sh new file mode 100644 index 000000000..19f59e6ae --- /dev/null +++ b/tests/build-remote-trustless-after.sh @@ -0,0 +1,2 @@ +outPath=$(readlink -f $TEST_ROOT/result) +grep 'FOO BAR BAZ' ${remoteDir}/${outPath} diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/build-remote-trustless-should-fail-0.sh new file mode 100644 index 000000000..5e3d5ae07 --- /dev/null +++ b/tests/build-remote-trustless-should-fail-0.sh @@ -0,0 +1,29 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +# We first build a dependency of the derivation we eventually want to +# build. +nix-build build-hook.nix -A passthru.input2 \ + -o "$TEST_ROOT/input2" \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ + --option system-features bar + +# Now when we go to build that downstream derivation, Nix will fail +# because we cannot trustlessly build input-addressed derivations with +# `inputDrv` dependencies. + +file=build-hook.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +expectStderr 1 source build-remote-trustless.sh \ + | grepQuiet "you are not privileged to build input-addressed derivations" diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/build-remote-trustless-should-pass-0.sh new file mode 100644 index 000000000..2a7ebd8c6 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-0.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-store +proto=ssh + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/build-remote-trustless-should-pass-1.sh new file mode 100644 index 000000000..516bdf092 --- /dev/null +++ b/tests/build-remote-trustless-should-pass-1.sh @@ -0,0 +1,9 @@ +source common.sh + +# Remote trusts us +file=build-hook.nix +prog=nix-daemon +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/build-remote-trustless-should-pass-3.sh new file mode 100644 index 000000000..40f81da5a --- /dev/null +++ b/tests/build-remote-trustless-should-pass-3.sh @@ -0,0 +1,14 @@ +source common.sh + +enableFeatures "daemon-trust-override" + +restartDaemon + +# Remote doesn't trusts us, but this is fine because we are only +# building (fixed) CA derivations. +file=build-hook-ca-fixed.nix +prog=$(readlink -e ./nix-daemon-untrusting.sh) +proto=ssh-ng + +source build-remote-trustless.sh +source build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless.sh b/tests/build-remote-trustless.sh new file mode 100644 index 000000000..9df44e0c5 --- /dev/null +++ b/tests/build-remote-trustless.sh @@ -0,0 +1,14 @@ +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +remoteDir=$TEST_ROOT/remote + +# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# more details. +nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ + --arg busybox $busybox \ + --store $TEST_ROOT/local \ + --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff --git a/tests/local.mk b/tests/local.mk index 6cb466e8e..7c3b42599 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -70,6 +70,10 @@ nix_tests = \ check-reqs.sh \ build-remote-content-addressed-fixed.sh \ build-remote-content-addressed-floating.sh \ + build-remote-trustless-should-pass-0.sh \ + build-remote-trustless-should-pass-1.sh \ + build-remote-trustless-should-pass-3.sh \ + build-remote-trustless-should-fail-0.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ diff --git a/tests/nix-daemon-untrusting.sh b/tests/nix-daemon-untrusting.sh new file mode 100755 index 000000000..bcdb70989 --- /dev/null +++ b/tests/nix-daemon-untrusting.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec nix-daemon --force-untrusted "$@" From aa74c7b0bcd31a6c0f75f5fa09f417bcbef4ad14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 17 Apr 2023 11:22:31 -0400 Subject: [PATCH 094/114] Gate experimental features in `DerivationOutput::fromJSON` This is an entry point for outside data, so we need to check enabled experimental features here. --- src/libstore/derivations.cc | 5 +++- src/libstore/derivations.hh | 6 ++++- src/libstore/tests/derivation.cc | 42 +++++++++++++++++++++++++------- tests/ca/derivation-json.sh | 3 +++ tests/impure-derivations.sh | 9 +++++++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0de36504b..15f3908ed 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -989,7 +989,8 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & _json) + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) { std::set keys; auto json = (std::map) _json; @@ -1028,6 +1029,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::CAFloating { .method = method, @@ -1040,6 +1042,7 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); auto [method, hashType] = methodAlgo(); return DerivationOutput::Impure { .method = method, diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index ccdde36ca..d00b23b6d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -136,11 +136,15 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + /** + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, std::string_view outputName, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 80ee52fd0..6f94904dd 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -1,6 +1,7 @@ #include #include +#include "experimental-features.hh" #include "derivations.hh" #include "tests/libstore.hh" @@ -9,10 +10,32 @@ namespace nix { class DerivationTest : public LibStoreTest { +public: + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; }; -#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ STR ## _json, \ @@ -22,7 +45,7 @@ class DerivationTest : public LibStoreTest OUTPUT_NAME)); \ } \ \ - TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ DerivationOutput { VAL }, \ @@ -30,10 +53,11 @@ class DerivationTest : public LibStoreTest *store, \ DRV_NAME, \ OUTPUT_NAME, \ - STR ## _json)); \ + STR ## _json, \ + mockXpSettings)); \ } -TEST_JSON(inputAddressed, +TEST_JSON(DerivationTest, inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -42,7 +66,7 @@ TEST_JSON(inputAddressed, }), "drv-name", "output-name") -TEST_JSON(caFixed, +TEST_JSON(DerivationTest, caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -56,7 +80,7 @@ TEST_JSON(caFixed, }), "drv-name", "output-name") -TEST_JSON(caFloating, +TEST_JSON(CaDerivationTest, caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -66,12 +90,12 @@ TEST_JSON(caFloating, }), "drv-name", "output-name") -TEST_JSON(deferred, +TEST_JSON(DerivationTest, deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(impure, +TEST_JSON(ImpureDerivationTest, impure, R"({ "hashAlgo": "r:sha256", "impure": true diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh index 3615177e9..c1480fd17 100644 --- a/tests/ca/derivation-json.sh +++ b/tests/ca/derivation-json.sh @@ -16,6 +16,9 @@ drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] +# But the JSON is rejected without the experimental feature +expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" + # Without --dry-run it is actually written drvPath4=$(nix derivation add < $TEST_HOME/foo.json) [[ "$drvPath4" = "$drvPath3" ]] diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index c7dadf397..39d053a04 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -10,6 +10,15 @@ clearStore # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter +# `nix derivation add` with impure derivations work +drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) +nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json +drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +[[ "$drvPath" = "$drvPath2" ]] + +# But only with the experimental feature! +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" + nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) path1=$(echo $json | jq -r .[].outputs.out) From 324ed0c36732fdc6c8230271da3f1a211b7ee8d4 Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Mon, 17 Apr 2023 20:15:08 -0700 Subject: [PATCH 095/114] Documentation: fix typo for `Nix database` link in manual Fixes broken link for `Nix database` anchor in the Glossary page of the Nix manual. --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a9782be5c..eeb19ad50 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -127,7 +127,7 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. - - Nix database{#gloss-nix-database}\ + - [Nix database]{#gloss-nix-database}\ An SQlite database to track [reference]s between [store object]s. This is an implementation detail of the [local store]. From 40fcb22313e65d1a57d0f6052ec046971ca07b8c Mon Sep 17 00:00:00 2001 From: Michael Utz Date: Tue, 18 Apr 2023 13:18:30 +0300 Subject: [PATCH 096/114] Update installing-binary.md --- doc/manual/src/installation/installing-binary.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index e3fd962bd..525654d35 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -136,7 +136,7 @@ which you may remove. ### macOS -1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing +1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: ```bash @@ -153,6 +153,7 @@ which you may remove. ```console sudo mv /etc/zshrc.backup-before-nix /etc/zshrc sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc ``` This will stop shells from sourcing the file and bringing everything you From d30d2dc861ddb20f035a0ae549e57fc439217b62 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Mon, 17 Apr 2023 19:34:09 +0200 Subject: [PATCH 097/114] Make "NAR info file is corrupt" messages more informative Recently, I encountered the "NAR info file 'xxxx' is corrupt" error with my binary cache. The message is not helpful in determining, which kind of corruption happened. The file, fetched with curl, looked reasonably. This commit adds more information to the error message, which should allow debugging and hopefully fixing the problem. --- src/libstore/nar-info.cc | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 274cd861c..d17253741 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,18 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { - auto corrupt = [&]() { - return Error("NAR info file '%1%' is corrupt", whence); + unsigned line = 1; + + auto corrupt = [&](const char * reason) { + return Error("NAR info file '%1%' is corrupt: %2%", whence, + std::string(reason) + (line > 0 ? " at line " + std::to_string(line) : "")); }; auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { - throw corrupt(); + throw corrupt("bad hash"); } }; @@ -26,12 +29,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); + if (colon == std::string::npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); + if (eol == std::string::npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -47,7 +50,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & fileHash = parseHashField(value); else if (name == "FileSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid FileSize"); fileSize = *n; } else if (name == "NarHash") { @@ -56,12 +59,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } else if (name == "NarSize") { auto n = string2Int(value); - if (!n) throw corrupt(); + if (!n) throw corrupt("invalid NarSize"); narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) throw corrupt(); + if (!references.empty()) throw corrupt("extra References"); for (auto & r : refs) references.insert(StorePath(r)); } @@ -72,17 +75,26 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (ca) throw corrupt(); + if (ca) throw corrupt("extra CA"); // FIXME: allow blank ca or require skipping field? ca = ContentAddress::parseOpt(value); } pos = eol + 1; + line += 1; } if (compression == "") compression = "bzip2"; - if (!havePath || !haveNarHash || url.empty() || narSize == 0) throw corrupt(); + if (!havePath || !haveNarHash || url.empty() || narSize == 0) { + line = 0; // don't include line information in the error + throw corrupt( + !havePath ? "StorePath missing" : + !haveNarHash ? "NarHash missing" : + url.empty() ? "URL missing" : + narSize == 0 ? "NarSize missing or zero" + : "?"); + } } std::string NarInfo::to_string(const Store & store) const From 5cd9890e8a96e2f2ab205738c1e2e4a6b615f443 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 18 Apr 2023 16:06:58 +0200 Subject: [PATCH 098/114] src/nix/flake.md: Itemize safe nixConfigs --- src/nix/flake.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 965f6eb48..456fd0ea1 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -381,10 +381,12 @@ The following attributes are supported in `flake.nix`: * `nixConfig`: a set of `nix.conf` options to be set when evaluating any part of a flake. In the interests of security, only a small set of - whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`, - `bash-prompt-suffix`, `flake-registry`, and `commit-lockfile-summary`) - are allowed to be set without confirmation so long as `accept-flake-config` - is not set in the global configuration. + set of options is allowed to be set without confirmation so long as [`accept-flake-config`](@docroot@/command-ref/conf-file.md#conf-accept-flake-config) is not enabled in the global configuration: + - [`bash-prompt`](@docroot@/command-ref/conf-file.md#conf-bash-prompt) + - [`bash-prompt-prefix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-prefix) + - [`bash-prompt-suffix`](@docroot@/command-ref/conf-file.md#conf-bash-prompt-suffix) + - [`flake-registry`](@docroot@/command-ref/conf-file.md#conf-flake-registry) + - [`commit-lockfile-summary`](@docroot@/command-ref/conf-file.md#conf-commit-lockfile-summary) ## Flake inputs From 3eb343754e42228b113bdcb7aec24ae18384a5fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Apr 2023 19:36:05 -0400 Subject: [PATCH 099/114] Move `test/recursive.sh` nix expr to file I found it hard to read as a big string literal. --- tests/recursive.nix | 56 +++++++++++++++++++++++++++++++++++++++++++ tests/recursive.sh | 58 +-------------------------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) create mode 100644 tests/recursive.nix diff --git a/tests/recursive.nix b/tests/recursive.nix new file mode 100644 index 000000000..fa8cc04db --- /dev/null +++ b/tests/recursive.nix @@ -0,0 +1,56 @@ +with import ./config.nix; + +mkDerivation rec { + name = "recursive"; + dummy = builtins.toFile "dummy" "bla bla"; + SHELL = shell; + + # Note: this is a string without context. + unreachable = builtins.getEnv "unreachable"; + + NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + mkdir $out + opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + # Check that we can query/build paths in our input closure. + nix $opts path-info $dummy + nix $opts build $dummy + + # Make sure we cannot query/build paths not in out input closure. + [[ -e $unreachable ]] + (! nix $opts path-info $unreachable) + (! nix $opts build $unreachable) + + # Add something to the store. + echo foobar > foobar + foobar=$(nix $opts store add-path ./foobar) + + nix $opts path-info $foobar + nix $opts build $foobar + + # Add it to our closure. + ln -s $foobar $out/foobar + + [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] + + # Build a derivation. + nix $opts build -L --impure --expr ' + with import ${./config.nix}; + mkDerivation { + name = "inner1"; + buildCommand = "echo $fnord blaat > $out"; + fnord = builtins.toFile "fnord" "fnord"; + } + ' + + [[ $(nix $opts path-info --json ./result) =~ fnord ]] + + ln -s $(nix $opts path-info ./result) $out/inner1 + ''; +} diff --git a/tests/recursive.sh b/tests/recursive.sh index 6335d44a5..638f06f85 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -12,63 +12,7 @@ rm -f $TEST_ROOT/result export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr ' - with import ./config.nix; - mkDerivation rec { - name = "recursive"; - dummy = builtins.toFile "dummy" "bla bla"; - SHELL = shell; - - # Note: this is a string without context. - unreachable = builtins.getEnv "unreachable"; - - NIX_TESTS_CA_BY_DEFAULT = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT"; - - requiredSystemFeatures = [ "recursive-nix" ]; - - buildCommand = '\'\'' - mkdir $out - opts="--experimental-features nix-command ${if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else ""}" - - PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH - - # Check that we can query/build paths in our input closure. - nix $opts path-info $dummy - nix $opts build $dummy - - # Make sure we cannot query/build paths not in out input closure. - [[ -e $unreachable ]] - (! nix $opts path-info $unreachable) - (! nix $opts build $unreachable) - - # Add something to the store. - echo foobar > foobar - foobar=$(nix $opts store add-path ./foobar) - - nix $opts path-info $foobar - nix $opts build $foobar - - # Add it to our closure. - ln -s $foobar $out/foobar - - [[ $(nix $opts path-info --all | wc -l) -eq 4 ]] - - # Build a derivation. - nix $opts build -L --impure --expr '\'' - with import ${./config.nix}; - mkDerivation { - name = "inner1"; - buildCommand = "echo $fnord blaat > $out"; - fnord = builtins.toFile "fnord" "fnord"; - } - '\'' - - [[ $(nix $opts path-info --json ./result) =~ fnord ]] - - ln -s $(nix $opts path-info ./result) $out/inner1 - '\'\''; - } -' +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] From 24005270ccbd5222023deb3306d5ee9a2f128cda Mon Sep 17 00:00:00 2001 From: Ezra Singh Date: Thu, 20 Apr 2023 18:36:01 -0400 Subject: [PATCH 100/114] Update install-systemd-multi-user.sh --- scripts/install-systemd-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 1a564d91d..0dfe11290 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -87,7 +87,7 @@ poly_configure_nix_daemon_service() { task "Setting up the nix-daemon systemd service" _sudo "to create the nix-daemon tmpfiles config" \ - ln -sfn /nix/var/nix/profiles/default$TMPFILES_SRC $TMPFILES_DEST + ln -sfn "/nix/var/nix/profiles/default$TMPFILES_SRC" "$TMPFILES_DEST" _sudo "to run systemd-tmpfiles once to pick that path up" \ systemd-tmpfiles --create --prefix=/nix/var/nix From 85f0cdc370021299142be9454483403e9bac2602 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 28 Jan 2023 20:31:10 -0500 Subject: [PATCH 101/114] Use `std::set` not `PathSet` for string contexts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Motivation `PathSet` is not correct because string contexts have other forms (`Built` and `DrvDeep`) that are not rendered as plain store paths. Instead of wrongly using `PathSet`, or "stringly typed" using `StringSet`, use `std::std`. ----- In support of this change, `NixStringContext` is now defined as `std::std` not `std:vector`. The old definition was just used by a `getContext` method which was only used by the eval cache. It can be deleted altogether since the types are now unified and the preexisting `copyContext` function already suffices. Summarizing the previous paragraph: Old: - `value/context.hh`: `NixStringContext = std::vector` - `value.hh`: `NixStringContext Value::getContext(...)` - `value.hh`: `copyContext(...)` New: - `value/context.hh`: `NixStringContext = std::set` - `value.hh`: `copyContext(...)` ---- The string representation of string context elements no longer contains the store dir. The diff of `src/libexpr/tests/value/context.cc` should make clear what the new representation is, so we recommend reviewing that file first. This was done for two reasons: Less API churn: `Value::mkString` and friends did not take a `Store` before. But if `NixStringContextElem::{parse, to_string}` *do* take a store (as they did before), then we cannot have the `Value` functions use them (in order to work with the fully-structured `NixStringContext`) without adding that argument. That would have been a lot of churn of threading the store, and this diff is already large enough, so the easier and less invasive thing to do was simply make the element `parse` and `to_string` functions not take the `Store` reference, and the easiest way to do that was to simply drop the store dir. Space usage: Dropping the `/nix/store/` (or similar) from the internal representation will safe space in the heap of the Nix programming being interpreted. If the heap contains many strings with non-trivial contexts, the saving could add up to something significant. ---- The eval cache version is bumped. The eval cache serialization uses `NixStringContextElem::{parse, to_string}`, and since those functions are changed per the above, that means the on-disk representation is also changed. This is simply done by changing the name of the used for the eval cache from `eval-cache-v4` to eval-cache-v5`. ---- To avoid some duplication `EvalCache::mkPathString` is added to abstract over the simple case of turning a store path to a string with just that string in the context. Context This PR picks up where #7543 left off. That one introduced the fully structured `NixStringContextElem` data type, but kept `PathSet context` as an awkward middle ground between internal `char[][]` interpreter heap string contexts and `NixStringContext` fully parsed string contexts. The infelicity of `PathSet context` was specifically called out during Nix team group review, but it was agreeing that fixing it could be left as future work. This is that future work. A possible follow-up step would be to get rid of the `char[][]` evaluator heap representation, too, but it is not yet clear how to do that. To use `NixStringContextElem` there we would need to get the STL containers to GC pointers in the GC build, and I am not sure how to do that. ---- PR #7543 effectively is writing the inverse of a `mkPathString`, `mkOutputString`, and one more such function for the `DrvDeep` case. I would like that PR to have property tests ensuring it is actually the inverse as expected. This PR sets things up nicely so that reworking that PR to be in that more elegant and better tested way is possible. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libcmd/installable-flake.cc | 6 +- src/libcmd/repl.cc | 4 +- src/libexpr/eval-cache.cc | 12 ++-- src/libexpr/eval.cc | 58 ++++++++-------- src/libexpr/eval.hh | 24 ++++--- src/libexpr/flake/flake.cc | 2 +- src/libexpr/get-drvs.cc | 6 +- src/libexpr/primops.cc | 95 +++++++++++++++----------- src/libexpr/primops/context.cc | 54 ++++++++------- src/libexpr/primops/fetchClosure.cc | 7 +- src/libexpr/primops/fetchMercurial.cc | 5 +- src/libexpr/primops/fetchTree.cc | 5 +- src/libexpr/tests/json.cc | 2 +- src/libexpr/tests/value/context.cc | 61 ++++++++--------- src/libexpr/value-to-json.cc | 6 +- src/libexpr/value-to-json.hh | 4 +- src/libexpr/value-to-xml.cc | 10 +-- src/libexpr/value-to-xml.hh | 2 +- src/libexpr/value.hh | 12 ++-- src/libexpr/value/context.cc | 17 +++-- src/libexpr/value/context.hh | 15 ++-- src/nix-env/nix-env.cc | 2 +- src/nix-env/user-env.cc | 6 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix/bundle.cc | 2 +- src/nix/eval.cc | 2 +- src/nix/flake.cc | 2 +- 27 files changed, 219 insertions(+), 204 deletions(-) diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index a3352af76..7e2a975f7 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -96,7 +96,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto v = attr->forceValue(); if (v.type() == nPath) { - PathSet context; + NixStringContext context; auto storePath = state->copyPathToStore(context, Path(v.path)); return {{ .path = DerivedPath::Opaque { @@ -107,10 +107,10 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } else if (v.type() == nString) { - PathSet context; + NixStringContext context; auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); auto storePath = state->store->maybeParseStorePath(s); - if (storePath && context.count(std::string(s))) { + if (storePath && context.count(NixStringContextElem::Opaque { .path = *storePath })) { return {{ .path = DerivedPath::Opaque { .path = std::move(*storePath), diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 80c08bf1c..8d404b04e 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -596,7 +596,7 @@ bool NixRepl::processLine(std::string line) const auto [path, line] = [&] () -> std::pair { if (v.type() == nPath || v.type() == nString) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); return {path, 0}; } else if (v.isLambda()) { @@ -940,7 +940,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m if (isDrv) { str << "«derivation "; Bindings::iterator i = v.attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i != v.attrs->end()) str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); else diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1219b2471..ba364f656 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString>(queryAttribute.getStr(3), ";")) - context.push_back(NixStringContextElem::parse(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -619,9 +619,11 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; - else if (v.type() == nPath) + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.string.s, std::move(context)}; + } else if (v.type() == nPath) return {v.path, {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6668add8c..56ff5908b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -609,8 +609,7 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } Path EvalState::checkSourcePath(const Path & path_) @@ -692,7 +691,7 @@ void EvalState::checkURI(const std::string & uri) } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -944,25 +943,25 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const PathSet & context) +static void copyContextToValue(Value & v, const NixStringContext & context) { if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); + v.string.context[n++] = dupString(i.to_string().c_str()); v.string.context[n] = 0; } } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); @@ -1038,6 +1037,16 @@ void EvalState::mkPos(Value & v, PosIdx p) } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -1900,7 +1909,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector s; size_t sSize = 0; NixInt n = 0; @@ -2109,26 +2118,15 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { if (v.string.context) for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); + context.insert(NixStringContextElem::parse(*p)); } -NixStringContext Value::getContext(const Store & store) -{ - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(NixStringContextElem::parse(store, *p)); - return res; -} - - -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); copyContext(v, context); @@ -2158,7 +2156,7 @@ bool EvalState::isDerivation(Value & v) std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) + NixStringContext & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { @@ -2172,7 +2170,7 @@ std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context, +BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, NixStringContext &context, std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); @@ -2249,7 +2247,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } -StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const Path & path) { if (nix::isDerivation(path)) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); @@ -2268,12 +2266,14 @@ StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) return dstPath; }(); - context.insert(store->printStorePath(dstPath)); + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +Path EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') @@ -2282,7 +2282,7 @@ Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) @@ -2489,7 +2489,7 @@ void EvalState::printStats() } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string", showType()) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b3b985683..74a7162ff 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -56,7 +56,7 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); -void copyContext(const Value & v, PathSet & context); +void copyContext(const Value & v, NixStringContext & context); /** @@ -327,7 +327,7 @@ public: * intended to distinguish between import-from-derivation and * sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + Path toRealPath(const Path & path, const NixStringContext & context); /** * Parse a Nix expression from the specified file. @@ -423,7 +423,7 @@ public: */ void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); [[gnu::noinline]] @@ -439,7 +439,7 @@ public: bool isDerivation(Value & v); std::optional tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); /** * String coercion. @@ -449,12 +449,12 @@ public: * booleans and lists to a string. If `copyToStore` is set, * referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(NixStringContext & context, const Path & path); /** * Path coercion. @@ -463,12 +463,12 @@ public: * path. The result is guaranteed to be a canonicalised, absolute * path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + Path coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** * Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); public: @@ -573,6 +573,12 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); + /* Create a string representing a store path. + + The string is the printed store path with a context containing a single + `Opaque` element of that store path. */ + void mkStorePathString(const StorePath & storePath, Value & v); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** @@ -584,7 +590,7 @@ public: * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context); private: diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ac396236f..3fb1f3536 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -265,7 +265,7 @@ static Flake getFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; + NixStringContext emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1602fbffb..506a63677 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -71,7 +71,7 @@ std::optional DrvInfo::queryDrvPath() const { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i == attrs->end()) drvPath = {std::nullopt}; else @@ -93,7 +93,7 @@ StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + NixStringContext context; if (i != attrs->end()) outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } @@ -124,7 +124,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; + NixStringContext context; outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 510f674eb..d11b56f8b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -38,17 +38,16 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; StringMap res; - for (auto & c_ : context) { + for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); }; - auto c = NixStringContextElem::parse(*store, c_); std::visit(overloaded { [&](const NixStringContextElem::Built & b) { drvs.push_back(DerivedPath::Built { @@ -112,7 +111,7 @@ struct RealisePathFlags { static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); @@ -158,7 +157,12 @@ static void mkOutputString( /* FIXME: we need to depend on the basic derivation, not derivation */ : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + NixStringContext { + NixStringContextElem::Built { + .drvPath = drvPath, + .output = o.first, + } + }); } /* Load and evaluate an expression from path specified by the @@ -181,7 +185,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v auto storePath = *optStorePath; Derivation drv = state.store->readDerivation(storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path, { + NixStringContextElem::DrvDeep { .drvPath = storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); @@ -358,7 +364,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); - PathSet context; + NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", false, false).toOwned(); @@ -768,7 +774,7 @@ static RegisterPrimOp primop_abort({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); @@ -787,7 +793,7 @@ static RegisterPrimOp primop_throw({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); state.debugThrowLastTrace(ThrownError(s)); @@ -800,7 +806,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; + NixStringContext context; auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); @@ -1086,7 +1092,7 @@ drvName, Bindings * attrs, Value & v) Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; @@ -1232,8 +1238,7 @@ drvName, Bindings * attrs, Value & v) /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & c_ : context) { - auto c = NixStringContextElem::parse(*state.store, c_); + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every path in the dependency graph of the derivation (including @@ -1392,7 +1397,9 @@ drvName, Bindings * attrs, Value & v) } auto result = state.buildBindings(1 + drv.outputs.size()); - result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) mkOutputString(state, result, drvPath, drv, i); @@ -1437,7 +1444,7 @@ static RegisterPrimOp primop_placeholder({ /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); v.mkString(canonPath(path), context); } @@ -1468,7 +1475,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; + NixStringContext context; Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so @@ -1482,7 +1489,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, auto path2 = state.store->toStorePath(path).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); + context.insert(NixStringContextElem::Opaque { .path = path2 }); v.mkString(path, context); } @@ -1538,7 +1545,7 @@ static RegisterPrimOp primop_pathExists({ following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), context); @@ -1560,7 +1567,7 @@ static RegisterPrimOp primop_baseNameOf({ of the argument. */ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.dirOf", false, false); @@ -1597,7 +1604,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V refsSink << s; refs = refsSink.getResultPaths(); } - auto context = state.store->printStorePathSet(refs); + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); + } v.mkString(s, context); } @@ -1628,7 +1640,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *i->value, context, "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", false, false).toOwned(); @@ -1787,7 +1799,7 @@ static RegisterPrimOp primop_readDir({ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1895,7 +1907,7 @@ static RegisterPrimOp primop_toXML({ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1945,22 +1957,23 @@ static RegisterPrimOp primop_fromJSON({ as an input by derivations. */ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') + for (auto c : context) { + if (auto p = std::get_if(&c)) + refs.insert(p->path); + else state.debugThrowLastTrace(EvalError({ .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", - name, path), + name, c.to_string()), .errPos = state.positions[pos] })); - refs.insert(state.store->parseStorePath(path)); } auto storePath = settings.readOnlyMode @@ -2061,7 +2074,7 @@ static void addPath( FileIngestionMethod method, const std::optional expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { // FIXME: handle CA derivation outputs (where path needs to @@ -2135,7 +2148,7 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); @@ -2204,7 +2217,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional expectedHash; - PathSet context; + NixStringContext context; for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; @@ -3538,7 +3551,7 @@ static RegisterPrimOp primop_lessThan({ `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false); @@ -3577,7 +3590,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, { int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) @@ -3611,7 +3624,7 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3637,7 +3650,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; // discarded + NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); @@ -3683,7 +3696,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; @@ -3763,7 +3776,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); @@ -3860,7 +3873,7 @@ static RegisterPrimOp primop_split({ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); @@ -3900,15 +3913,15 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector> to; + std::vector> to; to.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - PathSet ctx; + NixStringContext ctx; auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); to.emplace_back(s, std::move(ctx)); } - PathSet context; + NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db43e5771..07bf400cf 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } @@ -17,7 +17,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -33,17 +33,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto && p : context) { - auto c = NixStringContextElem::parse(*state.store, p); + NixStringContext context2; + for (auto && c : context) { if (auto * ptr = std::get_if(&c)) { - context2.emplace(state.store->printStorePath(ptr->drvPath)); + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); } else { /* Can reuse original item */ - context2.emplace(std::move(p)); + context2.emplace(std::move(c)); } } @@ -79,22 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, bool allOutputs = false; Strings outputs; }; - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); auto contextInfos = std::map(); - for (const auto & p : context) { - NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + for (auto && i : context) { std::visit(overloaded { - [&](NixStringContextElem::DrvDeep & d) { - contextInfos[d.drvPath].allOutputs = true; + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; }, - [&](NixStringContextElem::Built & b) { - contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output)); + [&](NixStringContextElem::Built && b) { + contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); }, - [&](NixStringContextElem::Opaque & o) { - contextInfos[o.path].path = true; + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; }, - }, ctx.raw()); + }, ((NixStringContextElem &&) i).raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -129,7 +129,7 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); @@ -143,13 +143,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .msg = hintfmt("context key '%s' is not a store path", name), .errPos = state.positions[i.pos] }); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(name)); + state.store->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) - context.emplace(name); + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } iter = i.value->attrs->find(sAllOutputs); @@ -161,7 +164,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); } - context.insert(concatStrings("=", name)); + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, + }); } } @@ -176,7 +181,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); - context.insert(concatStrings("!", outputName, "!", name)); + context.emplace(NixStringContextElem::Built { + .drvPath = namePath, + .output = std::string { outputName }, + }); } } } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 0dfa97fa3..4cf1f1e0b 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -18,7 +18,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg const auto & attrName = state.symbols[attr.name]; if (attrName == "fromPath") { - PathSet context; + NixStringContext context; fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } @@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg state.forceValue(*attr.value, attr.pos); toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; + NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } @@ -114,8 +114,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg }); } - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + state.mkStorePathString(*toPath, v); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c41bd60b6..2c0d98e74 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -13,7 +13,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a std::optional rev; std::optional ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -73,8 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 2e150c9d0..cd7039025 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -24,9 +24,8 @@ void emitTreeAttrs( auto attrs = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -107,7 +106,7 @@ static void fetchTree( const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 411bc0ac3..7586bdd9b 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -8,7 +8,7 @@ namespace nix { protected: std::string getJSONValue(Value& value) { std::stringstream ss; - PathSet ps; + NixStringContext ps; printValueAsJSON(state, true, value, noPos, ss, ps); return ss.str(); } diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 083359b7a..27d6920b0 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,69 +8,62 @@ namespace nix { -// Testing of trivial expressions -struct NixStringContextElemTest : public LibExprTest { - const Store & store() const { - return *LibExprTest::store; - } -}; - -TEST_F(NixStringContextElemTest, empty_invalid) { +TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), ""), + NixStringContextElem::parse(""), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, single_bang_invalid) { +TEST(NixStringContextElemTest, single_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!"), + NixStringContextElem::parse("!"), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, double_bang_invalid) { +TEST(NixStringContextElemTest, double_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!!/"), + NixStringContextElem::parse("!!/"), BadStorePath); } -TEST_F(NixStringContextElemTest, eq_slash_invalid) { +TEST(NixStringContextElemTest, eq_slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "=/"), + NixStringContextElem::parse("=/"), BadStorePath); } -TEST_F(NixStringContextElemTest, slash_invalid) { +TEST(NixStringContextElemTest, slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "/"), + NixStringContextElem::parse("/"), BadStorePath); } -TEST_F(NixStringContextElemTest, opaque) { - std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; - auto elem = NixStringContextElem::parse(store(), opaque); +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->path, store().parseStorePath(opaque)); - ASSERT_EQ(elem.to_string(store()), opaque); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); } -TEST_F(NixStringContextElemTest, drvDeep) { - std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), drvDeep); +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); auto * p = std::get_if(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); - ASSERT_EQ(elem.to_string(store()), drvDeep); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); } -TEST_F(NixStringContextElemTest, built) { - std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), built); +TEST(NixStringContextElemTest, built) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); - ASSERT_EQ(elem.to_string(store()), built); + ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(elem.to_string(), built); } } @@ -116,12 +109,12 @@ Gen Arbitrary::arbitrary() namespace nix { -RC_GTEST_FIXTURE_PROP( +RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { - RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index c35c876e3..67550b6b1 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,7 +11,7 @@ namespace nix { using json = nlohmann::json; json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); @@ -94,13 +94,13 @@ json printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore) const + NixStringContext & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 713356c7f..47ac90313 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { nlohmann::json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 341c8922f..fe652fd49 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,7 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos); @@ -32,7 +32,7 @@ static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; @@ -54,7 +54,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) { checkInterrupt(); @@ -166,7 +166,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -174,7 +174,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index ace7ead0f..6d702c0f2 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7739f99df..2bb791841 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -100,7 +100,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, @@ -112,13 +112,13 @@ class ExternalValueBase * Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore = true) const; + NixStringContext & context, bool copyToStore = true) const; /** * Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() @@ -268,9 +268,9 @@ public: void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); inline void mkPath(const char * s) { @@ -394,8 +394,6 @@ public: */ bool isTrivial() const; - NixStringContext getContext(const Store &); - auto listItems() { struct ListIterable diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 61d9c53df..f76fc76e4 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,11 +1,10 @@ #include "value/context.hh" -#include "store-api.hh" #include namespace nix { -NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +NixStringContextElem NixStringContextElem::parse(std::string_view s0) { std::string_view s = s0; @@ -25,41 +24,41 @@ NixStringContextElem NixStringContextElem::parse(const Store & store, std::strin "String content element beginning with '!' should have a second '!'"); } return NixStringContextElem::Built { - .drvPath = store.parseStorePath(s.substr(index + 1)), + .drvPath = StorePath { s.substr(index + 1) }, .output = std::string(s.substr(0, index)), }; } case '=': { return NixStringContextElem::DrvDeep { - .drvPath = store.parseStorePath(s.substr(1)), + .drvPath = StorePath { s.substr(1) }, }; } default: { return NixStringContextElem::Opaque { - .path = store.parseStorePath(s), + .path = StorePath { s }, }; } } } -std::string NixStringContextElem::to_string(const Store & store) const { +std::string NixStringContextElem::to_string() const { return std::visit(overloaded { [&](const NixStringContextElem::Built & b) { std::string res; res += '!'; res += b.output; res += '!'; - res += store.printStorePath(b.drvPath); + res += b.drvPath.to_string(); return res; }, [&](const NixStringContextElem::DrvDeep & d) { std::string res; res += '='; - res += store.printStorePath(d.drvPath); + res += d.drvPath.to_string(); return res; }, [&](const NixStringContextElem::Opaque & o) { - return store.printStorePath(o.path); + return std::string { o.path.to_string() }; }, }, raw()); } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 8719602d8..287ae08a9 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -26,8 +26,6 @@ public: } }; -class Store; - /** * Plain opaque path to some store object. * @@ -80,12 +78,15 @@ struct NixStringContextElem : _NixStringContextElem_Raw { using DrvDeep = NixStringContextElem_DrvDeep; using Built = NixStringContextElem_Built; - inline const Raw & raw() const { + inline const Raw & raw() const & { return static_cast(*this); } - inline Raw & raw() { + inline Raw & raw() & { return static_cast(*this); } + inline Raw && raw() && { + return static_cast(*this); + } /** * Decode a context string, one of: @@ -93,10 +94,10 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘=’ * - ‘!!’ */ - static NixStringContextElem parse(const Store & store, std::string_view s); - std::string to_string(const Store & store) const; + static NixStringContextElem parse(std::string_view s); + std::string to_string() const; }; -typedef std::vector NixStringContext; +typedef std::set NixStringContext; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index f076ffdb0..06f9ff6c4 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -960,7 +960,7 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); metaObj[j] = nullptr; } else { - PathSet context; + NixStringContext context; metaObj[j] = printValueAsJSON(*globals.state, true, *v, noPos, context); } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 745e9e174..3f9030827 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -119,9 +119,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Construct a Nix expression that calls the user environment builder with the manifest as argument. */ auto attrs = state.buildBindings(3); - attrs.alloc("manifest").mkString( - state.store->printStorePath(manifestFile), - {state.store->printStorePath(manifestFile)}); + state.mkStorePathString(manifestFile, attrs.alloc("manifest")); attrs.insert(state.symbols.create("derivations"), &manifest); Value args; args.mkAttrs(attrs); @@ -132,7 +130,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Evaluate it. */ debug("evaluating user environment builder"); state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); - PathSet context; + NixStringContext context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 6b5ba595d..4f1d12129 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -43,7 +43,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v, [&]() { return v.determinePos(noPos); }); - PathSet context; + NixStringContext context; if (evalOnly) { Value vRes; if (autoArgs.empty()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 57c355f0c..bcc00d490 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -98,7 +98,7 @@ struct CmdBundle : InstallableValueCommand if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - PathSet context2; + NixStringContext context2; auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); auto attr2 = vRes->attrs->get(evalState->sOutPath); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 43db5150c..ec0112e13 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -62,7 +62,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption auto state = getEvalState(); auto [v, pos] = installable->toValue(*state); - PathSet context; + NixStringContext context; if (apply) { auto vApply = state->allocValue(); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index cd4ee5921..40aabfdb3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -438,7 +438,7 @@ struct CmdFlakeCheck : FlakeCommand if (auto attr = v.attrs->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { - PathSet context; + NixStringContext context; auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); if (!store->isInStore(path)) throw Error("template '%s' has a bad 'path' attribute"); From ad57cff9bc38a4a97f4ecccc7655e2dfd73fc75c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Apr 2023 13:34:46 +0200 Subject: [PATCH 102/114] Document tMisc Co-authored-by: Robert Hensing --- src/libfetchers/input-accessor.hh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 23c510d4d..1cb233c51 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -13,7 +13,17 @@ class Store; struct InputAccessor { - enum Type { tRegular, tSymlink, tDirectory, tMisc }; + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; struct Stat { From 5d3f6dbf59554fb4701cb678a4ef29918db97767 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Apr 2023 13:37:51 +0200 Subject: [PATCH 103/114] Add some more SourcePath docs --- src/libfetchers/input-accessor.hh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 1cb233c51..5a2f17f62 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -125,9 +125,17 @@ struct SourcePath std::string to_string() const { return path.abs(); } + /** + * Append a `CanonPath` to this path. + */ SourcePath operator + (const CanonPath & x) const { return {path + x}; } + /** + * Append a single component `c` to this path. `c` must not + * contain a slash. A slash is implicitly added between this path + * and `c`. + */ SourcePath operator + (std::string_view c) const { return {path + c}; } From 880e7b8ed6d666519cc39512a8d1b117506a5264 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Jan 2023 23:14:48 +0100 Subject: [PATCH 104/114] TarArchive: Remove a duplicate constant and increase the buffer size --- src/libutil/tarfile.cc | 4 ++-- src/libutil/tarfile.hh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 238d0a7a6..5060a8f24 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -17,7 +17,7 @@ static ssize_t callback_read(struct archive * archive, void * _self, const void *buffer = self->buffer.data(); try { - return self->source->read((char *) self->buffer.data(), 4096); + return self->source->read((char *) self->buffer.data(), self->buffer.size()); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { @@ -39,7 +39,7 @@ void TarArchive::check(int err, const std::string & reason) throw Error(reason, archive_error_string(this->archive)); } -TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) +TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) { this->archive = archive_read_new(); this->source = &source; diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 24afb710a..237d18c31 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -24,6 +24,7 @@ struct TarArchive { ~TarArchive(); }; + void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); From a74d397549f987ee430be01f563e3fe9226ecb86 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Dec 2022 12:40:14 +0100 Subject: [PATCH 105/114] nix build --json: Only show non-zero startTime / stopTime --- src/nix/build.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 4e133e288..abf946214 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -27,8 +27,10 @@ nlohmann::json builtPathsWithResultToJSON(const std::vector std::visit([&](const auto & t) { auto j = t.toJSON(store); if (b.result) { - j["startTime"] = b.result->startTime; - j["stopTime"] = b.result->stopTime; + if (b.result->startTime) + j["startTime"] = b.result->startTime; + if (b.result->stopTime) + j["stopTime"] = b.result->stopTime; if (b.result->cpuUser) j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; if (b.result->cpuSystem) From 87f676b3a0dcc990c795abd93411f5b179373e60 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 25 Apr 2023 16:52:02 +0200 Subject: [PATCH 106/114] Formatting --- src/libexpr/eval-cache.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index ac646af8a..9e734e654 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -625,7 +625,8 @@ string_t AttrCursor::getStringWithContext() NixStringContext context; copyContext(v, context); return {v.string.s, std::move(context)}; - } else if (v.type() == nPath) + } + else if (v.type() == nPath) return {v.path().to_string(), {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); From f0d2b7eef3f85008163ae356fca92e67c8a90f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Christ?= Date: Wed, 26 Apr 2023 09:23:19 +0200 Subject: [PATCH 107/114] Doc: Improve builtins.genericClosure --- src/libexpr/primops.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cea5b4202..22ac780a5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -706,12 +706,14 @@ static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { .arity = 1, .doc = R"( Take an *attrset* with values named `startSet` and `operator` in order to - return a *list of attrsets* by starting with the `startSet`, recursively - applying the `operator` function to each element. The *attrsets* in the - `startSet` and produced by the `operator` must each contain value named - `key` which are comparable to each other. The result is produced by - repeatedly calling the operator for each element encountered with a - unique key, terminating when no new elements are produced. For example, + return a *list of attrsets* by starting with the `startSet` and recursively + applying the `operator` function to each `item`. The *attrsets* in the + `startSet` and the *attrsets* produced by `operator` must contain a value + named `key` which is comparable. The result is produced by calling `operator` + for each `item` with a value for `key` that has not been called yet including + newly produced `item`s. The function terminates when no new `item`s are + produced. The resulting *list of attrsets* contains only *attrsets* with a + unique key. For example, ``` builtins.genericClosure { From 9b2a4a472904a20faa2baec183efbace9f7a5519 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 28 Apr 2023 11:53:37 +0200 Subject: [PATCH 108/114] move uninstall instructions to a separate page placed in a subsection of the binary install, the instructions are hard to find. putting them in a separate page that is shown in the table of contents should make it easier for users to find what they need when they need it. --- doc/manual/src/SUMMARY.md.in | 1 + .../src/installation/installing-binary.md | 168 +----------------- doc/manual/src/installation/uninstall.md | 159 +++++++++++++++++ 3 files changed, 166 insertions(+), 162 deletions(-) create mode 100644 doc/manual/src/installation/uninstall.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f783d5908..1d5613354 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -15,6 +15,7 @@ - [Multi-User Mode](installation/multi-user.md) - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) + - [Uninstalling Nix](installation/uninstall.md) - [Package Management](package-management/package-management.md) - [Basic Package Management](package-management/basic-package-mgmt.md) - [Profiles](package-management/profiles.md) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 525654d35..ffabb250a 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -47,12 +47,6 @@ The install script will modify the first writable file from amongst `NIX_INSTALLER_NO_MODIFY_PROFILE` environment variable before executing the install script to disable this behaviour. -You can uninstall Nix simply by running: - -```console -$ rm -rf /nix -``` - # Multi User Installation The multi-user Nix installation creates system users, and a system @@ -84,155 +78,8 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist. The installer will first back up these files with a `.backup-before-nix` extension. The installer will also create `/etc/profile.d/nix.sh`. -## Uninstalling - -### Linux - -If you are on Linux with systemd: - -1. Remove the Nix daemon service: - - ```console - sudo systemctl stop nix-daemon.service - sudo systemctl disable nix-daemon.socket nix-daemon.service - sudo systemctl daemon-reload - ``` - -1. Remove systemd service files: - - ```console - sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket - ``` - -1. The installer script uses systemd-tmpfiles to create the socket directory. - You may also want to remove the configuration for that: - - ```console - sudo rm /etc/tmpfiles.d/nix-daemon.conf - ``` - -Remove files created by Nix: - -```console -sudo rm -rf /nix /etc/nix /etc/profile/nix.sh ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels -``` - -Remove build users and their group: - -```console -for i in $(seq 1 32); do - sudo userdel nixbld$i -done -sudo groupdel nixbld -``` - -There may also be references to Nix in - -- `/etc/profile` -- `/etc/bashrc` -- `/etc/zshrc` - -which you may remove. - -### macOS - -1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing - `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - - If these files haven't been altered since installing Nix you can simply put - the backups back in place: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - This will stop shells from sourcing the file and bringing everything you - installed using Nix in scope. - -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you - boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store - volume on `/nix`, which looks like - `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or - `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic - mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only - line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. - This will prevent the creation of the empty `/nix` directory to provide a - mountpoint for the Nix Store volume. - -6. Remove the files Nix added to your system: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - ``` - - This gets rid of any data Nix may have created except for the store which is - removed next. - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the - store. - - If the output indicates that the command couldn't remove the volume, you should - make sure you don't have an _unmounted_ Nix Store volume. Look for a - "Nix Store" volume in the output of the following command: - - ```console - diskutil list - ``` - - If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil - deleteVolume command, but replace `/nix` with the store volume's `diskXsY` - identifier. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` -> directory. This is an expected sign of a successful uninstall. The empty -> `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. The uninstall is -> complete. macOS (Catalina+) directly controls root directories and its -> read-only root will prevent you from manually deleting the empty `/nix` -> mountpoint. - # macOS Installation + []{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} @@ -281,19 +128,16 @@ this to run the installer, but it may help if you run into trouble: # Installing a pinned Nix version from a URL -NixOS.org hosts version-specific installation URLs for all Nix versions -since 1.11.16, at `https://releases.nixos.org/nix/nix-version/install`. +Version-specific installation URLs for all Nix versions +since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). +The corresponding SHA-256 hash can be found in the directory for the given version. -These install scripts can be used the same as the main NixOS.org -installation script: +These install scripts can be used the same as usual: ```console -$ curl -L https://nixos.org/nix/install | sh +$ curl -L https://releases.nixos.org/nix/nix-/install | sh ``` -In the same directory of the install script are sha256 sums, and gpg -signature files. - # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md new file mode 100644 index 000000000..bd85b49ee --- /dev/null +++ b/doc/manual/src/installation/uninstall.md @@ -0,0 +1,159 @@ +# Uninstalling Nix + +## Single User + +If you have a [single-user installation](./installing-binary.md#single-user-installation) of Nix, uninstall it by running: + +```console +$ rm -rf /nix +``` + +## Multi User + +Removing a [multi-user installation](./installing-binary.md#multi-user-installation) of Nix is more involved, and depends on the operating system. + +### Linux + +If you are on Linux with systemd: + +1. Remove the Nix daemon service: + + ```console + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket nix-daemon.service + sudo systemctl daemon-reload + ``` + +1. Remove systemd service files: + + ```console + sudo rm /etc/systemd/system/nix-daemon.service /etc/systemd/system/nix-daemon.socket + ``` + +1. The installer script uses systemd-tmpfiles to create the socket directory. + You may also want to remove the configuration for that: + + ```console + sudo rm /etc/tmpfiles.d/nix-daemon.conf + ``` + +Remove files created by Nix: + +```console +sudo rm -rf /nix /etc/nix /etc/profile/nix.sh ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels +``` + +Remove build users and their group: + +```console +for i in $(seq 1 32); do + sudo userdel nixbld$i +done +sudo groupdel nixbld +``` + +There may also be references to Nix in + +- `/etc/profile` +- `/etc/bashrc` +- `/etc/zshrc` + +which you may remove. + +### macOS + +1. Edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: + + ```bash + # Nix + if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' + fi + # End Nix + ``` + + If these files haven't been altered since installing Nix you can simply put + the backups back in place: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc + ``` + + This will stop shells from sourcing the file and bringing everything you + installed using Nix in scope. + +2. Stop and remove the Nix daemon services: + + ```console + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist + sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist + ``` + + This stops the Nix daemon and prevents it from being started next time you + boot the system. + +3. Remove the `nixbld` group and the `_nixbuildN` users: + + ```console + sudo dscl . -delete /Groups/nixbld + for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done + ``` + + This will remove all the build users that no longer serve a purpose. + +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store + volume on `/nix`, which looks like + `UUID= /nix apfs rw,noauto,nobrowse,suid,owners` or + `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic + mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only + line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. + This will prevent the creation of the empty `/nix` directory to provide a + mountpoint for the Nix Store volume. + +6. Remove the files Nix added to your system: + + ```console + sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + ``` + + This gets rid of any data Nix may have created except for the store which is + removed next. + +7. Remove the Nix Store volume: + + ```console + sudo diskutil apfs deleteVolume /nix + ``` + + This will remove the Nix Store volume and everything that was added to the + store. + + If the output indicates that the command couldn't remove the volume, you should + make sure you don't have an _unmounted_ Nix Store volume. Look for a + "Nix Store" volume in the output of the following command: + + ```console + diskutil list + ``` + + If you _do_ see a "Nix Store" volume, delete it by re-running the diskutil + deleteVolume command, but replace `/nix` with the store volume's `diskXsY` + identifier. + +> **Note** +> +> After you complete the steps here, you will still have an empty `/nix` +> directory. This is an expected sign of a successful uninstall. The empty +> `/nix` directory will disappear the next time you reboot. +> +> You do not have to reboot to finish uninstalling Nix. The uninstall is +> complete. macOS (Catalina+) directly controls root directories and its +> read-only root will prevent you from manually deleting the empty `/nix` +> mountpoint. + From 17e6b85d05b3d32df244b1d4e89aa41fd8bdcae8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 28 Apr 2023 16:57:37 +0200 Subject: [PATCH 109/114] nix: Support the --repair flag --- src/libcmd/command.cc | 2 ++ src/libcmd/common-eval-args.hh | 3 ++- src/libmain/common-args.hh | 18 ++++++++++++++++++ src/nix-build/nix-build.cc | 9 ++------- src/nix-env/nix-env.cc | 5 +---- src/nix-instantiate/nix-instantiate.cc | 5 +---- src/nix/build.cc | 3 ++- 7 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index bedf11e2c..6c4648b34 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -121,6 +121,8 @@ ref EvalCommand::getEvalState() #endif ; + evalState->repair = repair; + if (startReplOnEvalErrors) { evalState->debugRepl = &AbstractNixRepl::runSimple; }; diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 83edcfb85..b65cb5b20 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "common-args.hh" namespace nix { @@ -10,7 +11,7 @@ class EvalState; class Bindings; struct SourcePath; -struct MixEvalArgs : virtual Args +struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index e7ed0d934..c35406c3b 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "repair-flag.hh" namespace nix { @@ -49,4 +50,21 @@ struct MixJSON : virtual Args } }; +struct MixRepair : virtual Args +{ + RepairFlag repair = NoRepair; + + MixRepair() + { + addFlag({ + .longName = "repair", + .description = + "During evaluation, rewrite missing or corrupted files in the Nix store. " + "During building, rebuild missing or corrupted store paths.", + .category = miscCategory, + .handler = {&repair, Repair}, + }); + } +}; + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 251f00edf..6510df8f0 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -84,7 +84,6 @@ static void main_nix_build(int argc, char * * argv) auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; Strings left; - RepairFlag repair = NoRepair; BuildMode buildMode = bmNormal; bool readStdin = false; @@ -169,11 +168,6 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--dry-run") dryRun = true; - else if (*arg == "--repair") { - repair = Repair; - buildMode = bmRepair; - } - else if (*arg == "--run-env") // obsolete runEnv = true; @@ -249,7 +243,8 @@ static void main_nix_build(int argc, char * * argv) auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + state->repair = myArgs.repair; + if (myArgs.repair) buildMode = bmRepair; auto autoArgs = myArgs.getAutoArgs(*state); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 8b3f903f6..5e94f2d14 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1391,7 +1391,6 @@ static int main_nix_env(int argc, char * * argv) Operation op = 0; std::string opName; bool showHelp = false; - RepairFlag repair = NoRepair; std::string file; Globals globals; @@ -1489,8 +1488,6 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.systemFilter = getArg(*arg, arg, end); else if (*arg == "--prebuilt-only" || *arg == "-b") globals.prebuiltOnly = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg != "" && arg->at(0) == '-') { opFlags.push_back(*arg); /* FIXME: hacky */ @@ -1515,7 +1512,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); - globals.state->repair = repair; + globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( file != "" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index fa6cc2bd7..446b27e66 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -102,7 +102,6 @@ static int main_nix_instantiate(int argc, char * * argv) bool strict = false; Strings attrPaths; bool wantsReadWrite = false; - RepairFlag repair = NoRepair; struct MyArgs : LegacyArgs, MixEvalArgs { @@ -140,8 +139,6 @@ static int main_nix_instantiate(int argc, char * * argv) xmlOutputSourceLocation = false; else if (*arg == "--strict") strict = true; - else if (*arg == "--repair") - repair = Repair; else if (*arg == "--dry-run") settings.readOnlyMode = true; else if (*arg != "" && arg->at(0) == '-') @@ -160,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; auto state = std::make_unique(myArgs.searchPath, evalStore, store); - state->repair = repair; + state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/build.cc b/src/nix/build.cc index abf946214..ad1842a4e 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -133,7 +133,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile auto buildables = Installable::build( getEvalStore(), store, Realise::Outputs, - installables, buildMode); + installables, + repair ? bmRepair : buildMode); if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); From 521cca1840fd0e2a240b09f732fdfc449d424f64 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 2 May 2023 11:28:03 +0200 Subject: [PATCH 110/114] add procedure for 'idea approved' label --- maintainers/README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/maintainers/README.md b/maintainers/README.md index 618bfb4e4..12858227f 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -77,10 +77,18 @@ Items on the board progress through the following states: - [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) Team members can also add pull requests or issues they would like the whole team to consider. - - If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. - To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. + + If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_. + Otherwise, the issue or pull request in questions get the label [`idea approved`](https://github.com/NixOS/nix/labels/idea%20approved). + For issues this means that an implementation is welcome and will be prioritised for review. + For pull requests this means that: + - Unfinished work is encouraged to be continued. + - A reviewer is assigned to take responsibility for getting the pull request merged. + The item is moved to the _Assigned_ column. + - If needed, the team can decide to do a collarorative review. + Then the item is moved to the _In review_ column, and review session is scheduled. + What constitutes a trivial pull request is up to maintainers' judgement. - To discuss @@ -110,12 +118,12 @@ Items on the board progress through the following states: When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. -- Assigned for merging +- Assigned One team member is assigned to each of these pull requests. They will communicate with the authors, and make the final approval once all remaining issues are addressed. - If more substantive issues arise, the assignee can move the pull request back to _To discuss_ to involve the team again. + If more substantive issues arise, the assignee can move the pull request back to _To discuss_ or _In review_ to involve the team again. The process is illustrated in the following diagram: From feb2200ba6b75c036f89cee3aff44b8fe09fc0a6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 2 May 2023 11:30:26 +0200 Subject: [PATCH 111/114] use headings instead of list items this allows easier linking and a provides a bit more visual clarity --- maintainers/README.md | 94 ++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/maintainers/README.md b/maintainers/README.md index 12858227f..d13349438 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -42,12 +42,12 @@ The team meets twice a week: - Discussion meeting: [Fridays 13:00-14:00 CET](https://calendar.google.com/calendar/event?eid=MHNtOGVuNWtrZXNpZHR2bW1sM3QyN2ZjaGNfMjAyMjExMjVUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) - 1. Triage issues and pull requests from the _No Status_ column (30 min) - 2. Discuss issues and pull requests from the _To discuss_ column (30 min) + 1. Triage issues and pull requests from the [No Status](#no-status) column (30 min) + 2. Discuss issues and pull requests from the [To discuss](#to-discuss) column (30 min) - Work meeting: [Mondays 13:00-15:00 CET](https://calendar.google.com/calendar/event?eid=NTM1MG1wNGJnOGpmOTZhYms3bTB1bnY5cWxfMjAyMjExMjFUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) - 1. Code review on pull requests from _In review_. + 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). @@ -58,72 +58,74 @@ The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19 Items on the board progress through the following states: -- No Status +### No Status - During the discussion meeting, the team triages new items. - To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance. +During the discussion meeting, the team triages new items. +To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance. - On every meeting, at least one item from each of the following categories is inspected: +On every meeting, at least one item from each of the following categories is inspected: - 1. [critical](https://github.com/NixOS/nix/labels/critical) - 2. [security](https://github.com/NixOS/nix/labels/security) - 3. [regression](https://github.com/NixOS/nix/labels/regression) - 4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) - 5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) +1. [critical](https://github.com/NixOS/nix/labels/critical) +2. [security](https://github.com/NixOS/nix/labels/security) +3. [regression](https://github.com/NixOS/nix/labels/regression) +4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) +5. [tests of existing functionality](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Atests+-label%3Afeature+sort%3Areactions-%2B1-desc) - - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) - - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) - - [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc) - - [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) +- [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) +- [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) +- [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc) +- [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) - Team members can also add pull requests or issues they would like the whole team to consider. - To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. +Team members can also add pull requests or issues they would like the whole team to consider. +To ensure process quality and reliability, all non-trivial pull requests must be triaged before merging. - If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_. - Otherwise, the issue or pull request in questions get the label [`idea approved`](https://github.com/NixOS/nix/labels/idea%20approved). - For issues this means that an implementation is welcome and will be prioritised for review. - For pull requests this means that: - - Unfinished work is encouraged to be continued. - - A reviewer is assigned to take responsibility for getting the pull request merged. - The item is moved to the _Assigned_ column. - - If needed, the team can decide to do a collarorative review. - Then the item is moved to the _In review_ column, and review session is scheduled. +If there is disagreement on the general idea behind an issue or pull request, it is moved to [To discuss](#to-discuss). +Otherwise, the issue or pull request in questions get the label [`idea approved`](https://github.com/NixOS/nix/labels/idea%20approved). +For issues this means that an implementation is welcome and will be prioritised for review. +For pull requests this means that: +- Unfinished work is encouraged to be continued. +- A reviewer is assigned to take responsibility for getting the pull request merged. + The item is moved to the [Assigned](#assigned) column. +- If needed, the team can decide to do a collarorative review. + Then the item is moved to the [In review](#in-review) column, and review session is scheduled. - What constitutes a trivial pull request is up to maintainers' judgement. +What constitutes a trivial pull request is up to maintainers' judgement. -- To discuss +### To discuss - Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. +Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. - This may be where the merit of the change itself or the implementation strategy is contested by a team member. +This may be where the merit of the change itself or the implementation strategy is contested by a team member. - As a general guideline, the order of items is determined as follows: +As a general guideline, the order of items is determined as follows: - - Prioritise pull requests over issues +- Prioritise pull requests over issues - Contributors who took the time to implement concrete change proposals should not wait indefinitely. + Contributors who took the time to implement concrete change proposals should not wait indefinitely. - - Prioritise fixing bugs and testing over documentation, improvements or new features +- Prioritise fixing bugs and testing over documentation, improvements or new features - The team values stability and accessibility higher than raw functionality. + The team values stability and accessibility higher than raw functionality. - - Interleave issues and PRs +- Interleave issues and PRs - This way issues without attempts at a solution get a chance to get addressed. + This way issues without attempts at a solution get a chance to get addressed. -- In review +### In review - Pull requests in this column are reviewed together during work meetings. - This is both for spreading implementation knowledge and for establishing common values in code reviews. +Pull requests in this column are reviewed together during work meetings. +This is both for spreading implementation knowledge and for establishing common values in code reviews. - When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. +When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. -- Assigned +### Assigned - One team member is assigned to each of these pull requests. - They will communicate with the authors, and make the final approval once all remaining issues are addressed. +One team member is assigned to each of these pull requests. +They will communicate with the authors, and make the final approval once all remaining issues are addressed. - If more substantive issues arise, the assignee can move the pull request back to _To discuss_ or _In review_ to involve the team again. +If more substantive issues arise, the assignee can move the pull request back to [To discuss](#to-discuss) or [In review](#in-review) to involve the team again. + +### Flowchart The process is illustrated in the following diagram: From 5d78dc41760c504240101e77eb38c47fec58bdbd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 May 2023 09:16:29 +0200 Subject: [PATCH 112/114] doc rendering: add functions to scope explicitly (#7378) * doc rendering: add functions to scope explicitly this especially helps beginners with code readability, since the origin of names is always immediately visible. --- doc/manual/generate-builtins.nix | 12 ++++++++---- doc/manual/generate-manpage.nix | 22 +++++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 115bb3f94..71f96153f 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,8 +1,12 @@ -builtinsDump: +let + inherit (builtins) concatStringsSep attrNames; +in + +builtinsInfo: let showBuiltin = name: let - inherit (builtinsDump.${name}) doc args; + inherit (builtinsInfo.${name}) doc args; in ''
@@ -14,7 +18,7 @@ let ''; - listArgs = args: builtins.concatStringsSep " " (map (s: "${s}") args); + listArgs = args: concatStringsSep " " (map (s: "${s}") args); in -with builtins; concatStringsSep "\n" (map showBuiltin (attrNames builtinsDump)) +concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo)) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index d04eecf55..fb34898f3 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,10 +1,16 @@ -cliDumpStr: +let + inherit (builtins) + attrNames attrValues fromJSON listToAttrs mapAttrs + concatStringsSep concatMap length lessThan replaceStrings sort; + inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; +in -with builtins; -with import ./utils.nix; +commandDump: let + commandInfo = fromJSON commandDump; + showCommand = { command, details, filename, toplevel }: let @@ -96,7 +102,7 @@ let ${option.description} ''; - categories = sort builtins.lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); + categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); in concatStrings (map showCategory categories); in squash result; @@ -117,13 +123,11 @@ let }; in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {}); - cliDump = builtins.fromJSON cliDumpStr; - manpages = processCommand { command = "nix"; - details = cliDump.args; + details = commandInfo.args; filename = "nix"; - toplevel = cliDump.args; + toplevel = commandInfo.args; }; tableOfContents = let @@ -143,6 +147,6 @@ let ${showSettings { useAnchors = false; } settings} ''; - in concatStrings (attrValues (mapAttrs showStore cliDump.stores)); + in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } From dc8191ae1457d091c14deb4c819479b68baa88d1 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 May 2023 11:39:29 +0200 Subject: [PATCH 113/114] add redirect to track moved uninstall section --- doc/manual/redirects.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 69f75d3a0..5cd6fdea2 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -338,6 +338,9 @@ const redirects = { "strings": "#string", "lists": "#list", "attribute-sets": "#attribute-set" + }, + "installation/installing-binary.html": { + "uninstalling": "uninstall.html" } }; From 0662fd859935b74efa62a4c56aff85cb749c17c5 Mon Sep 17 00:00:00 2001 From: figsoda Date: Wed, 3 May 2023 18:59:44 -0400 Subject: [PATCH 114/114] Fix hostRegex to accept hosts with a `-` --- src/libfetchers/github.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1ed09d30d..6c1d573ce 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -21,7 +21,7 @@ struct DownloadUrl }; // A github, gitlab, or sourcehut host -const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +const static std::string hostRegexS = "[a-zA-Z0-9.-]*"; // FIXME: check std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme