From 1d28d613b1d0447b60de2d2044aeac3e1d543aa6 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Wed, 25 Oct 2023 11:39:18 +0200 Subject: [PATCH 01/54] config: add included files into parsedContents before applying Fixes #8719 --- src/libutil/config.cc | 16 ++++++++++++---- tests/functional/init.sh | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..17380b6d8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -88,10 +88,9 @@ void Config::getSettings(std::map & res, bool overridd res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { - unsigned int pos = 0; - std::vector> parsedContents; +static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { + unsigned int pos = 0; while (pos < contents.size()) { std::string line; @@ -123,7 +122,10 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { - applyConfigFile(p); + try { + std::string includedContents = readFile(path); + applyConfigInner(includedContents, p, parsedContents); + } catch (SysError &) { } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } @@ -143,6 +145,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string concatStringsSep(" ", Strings(i, tokens.end())), }); }; +} + +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + std::vector> parsedContents; + + applyConfigInner(contents, path, parsedContents); // First apply experimental-feature related settings for (auto & [name, value] : parsedContents) diff --git a/tests/functional/init.sh b/tests/functional/init.sh index c420e8c9f..d697b1a30 100755 --- a/tests/functional/init.sh +++ b/tests/functional/init.sh @@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < Date: Fri, 6 Jan 2023 18:06:03 -0500 Subject: [PATCH 02/54] Factor out `StoreDirConfig` More progress on #5729. --- src/libstore/path.cc | 14 ++-- src/libstore/store-api.cc | 24 +++--- src/libstore/store-api.hh | 103 +------------------------ src/libstore/store-dir-config.hh | 126 +++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 119 deletions(-) create mode 100644 src/libstore/store-dir-config.hh diff --git a/src/libstore/path.cc b/src/libstore/path.cc index ec3e53232..69f6d7356 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,4 @@ -#include "store-api.hh" +#include "store-dir-config.hh" #include @@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(hash, name); } -StorePath Store::parseStorePath(std::string_view path) const +StorePath StoreDirConfig::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); if (dirOf(p) != storeDir) @@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional Store::maybeParseStorePath(std::string_view path) const +std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -71,24 +71,24 @@ std::optional Store::maybeParseStorePath(std::string_view path) const } } -bool Store::isStorePath(std::string_view path) const +bool StoreDirConfig::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet Store::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string Store::printStorePath(const StorePath & path) const +std::string StoreDirConfig::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet Store::printStorePathSet(const StorePathSet & paths) const +PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 646b0ec7d..a681bb6cf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -23,13 +23,13 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(PathView path) const +bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -143,7 +143,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const */ -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -154,14 +154,14 @@ StorePath Store::makeStorePath(std::string_view type, } -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath Store::makeOutputPath(std::string_view id, +StorePath StoreDirConfig::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -172,7 +172,7 @@ StorePath Store::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const Store & store, + const StoreDirConfig & store, std::string && type, const StoreReferences & references) { @@ -185,7 +185,7 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); @@ -201,7 +201,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf } -StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { assert(info.hash.type == htSHA256); return makeStorePath( @@ -214,7 +214,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -228,7 +228,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathFromDump( +std::pair StoreDirConfig::computeStorePathFromDump( Source & dump, std::string_view name, FileIngestionMethod method, @@ -247,7 +247,7 @@ std::pair Store::computeStorePathFromDump( } -StorePath Store::computeStorePathForText( +StorePath StoreDirConfig::computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const @@ -1315,7 +1315,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string Store::showPaths(const StorePathSet & paths) +std::string StoreDirConfig::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6aa317e3d..bee5ec16c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -14,6 +14,7 @@ #include "config.hh" #include "path-info.hh" #include "repair-flag.hh" +#include "store-dir-config.hh" #include #include @@ -64,7 +65,6 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); @@ -97,11 +97,11 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public Config +struct StoreConfig : public StoreDirConfig { typedef std::map Params; - using Config::Config; + using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; @@ -131,15 +131,6 @@ struct StoreConfig : public Config return std::nullopt; } - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; @@ -224,45 +215,6 @@ public: virtual std::string getUri() = 0; - StorePath parseStorePath(std::string_view path) const; - - std::optional maybeParseStorePath(std::string_view path) const; - - std::string printStorePath(const StorePath & path) const; - - /** - * Deprecated - * - * \todo remove - */ - StorePathSet parseStorePathSet(const PathSet & paths) const; - - PathSet printStorePathSet(const StorePathSet & path) const; - - /** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ - std::string showPaths(const StorePathSet & paths); - - /** - * @return true if ‘path’ is in the Nix store (but not the Nix - * store itself). - */ - bool isInStore(PathView path) const; - - /** - * @return true if ‘path’ is a store path, i.e. a direct child of the - * Nix store. - */ - bool isStorePath(std::string_view path) const; - - /** - * Split a path like /nix/store/-/ into - * /nix/store/- and /. - */ - std::pair toStorePath(PathView path) const; - /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -274,55 +226,6 @@ public: */ StorePath followLinksToStorePath(std::string_view path) const; - /** - * Constructs a unique store path name. - */ - StorePath makeStorePath(std::string_view type, - std::string_view hash, std::string_view name) const; - StorePath makeStorePath(std::string_view type, - const Hash & hash, std::string_view name) const; - - StorePath makeOutputPath(std::string_view id, - const Hash & hash, std::string_view name) const; - - StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; - - /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. - */ - std::pair computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - const StorePathSet & references = {}) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; - /** * Check whether a path is valid. */ diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh new file mode 100644 index 000000000..53843d663 --- /dev/null +++ b/src/libstore/store-dir-config.hh @@ -0,0 +1,126 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + +MakeError(BadStorePath, Error); + +struct StoreDirConfig : public Config +{ + using Config::Config; + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; + const Path storeDir = storeDir_; + + // pure methods + + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + /** + * Deprecated + * + * \todo remove + */ + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ + std::string showPaths(const StorePathSet & paths); + + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; + + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ + bool isStorePath(std::string_view path) const; + + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; + + /** + * Constructs a unique store path name. + */ + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const; + + StorePath makeOutputPath(std::string_view id, + const Hash & hash, std::string_view name) const; + + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. + */ + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; + + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ + StorePath computeStorePathForText( + std::string_view name, + std::string_view s, + const StorePathSet & references) const; +}; + +} From dde1d863388617b3a63db808c125f274c86a3222 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 15:35:45 +0000 Subject: [PATCH 03/54] Restrict some code to `StoreDirConfig` - part of eval cache - part of derivations - derived path - store path with outputs - serializers --- perl/lib/Nix/Store.xs | 1 + src/libcmd/built-path.cc | 8 ++-- src/libcmd/built-path.hh | 18 ++++----- src/libexpr/eval-cache.cc | 6 +-- src/libexpr/primops/fetchClosure.cc | 1 + src/libexpr/value.hh | 1 - src/libstore/builtins/buildenv.cc | 1 + src/libstore/builtins/buildenv.hh | 1 - src/libstore/common-protocol-impl.hh | 4 +- src/libstore/common-protocol.cc | 28 ++++++------- src/libstore/common-protocol.hh | 8 ++-- src/libstore/derivations.cc | 32 +++++++-------- src/libstore/derivations.hh | 26 +++++++------ src/libstore/derived-path.cc | 39 ++++++++++--------- src/libstore/derived-path.hh | 37 ++++++++++-------- .../length-prefixed-protocol-helper.hh | 22 +++++------ src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 1 + src/libstore/path-with-outputs.cc | 4 +- src/libstore/path-with-outputs.hh | 10 +++-- src/libstore/serve-protocol-impl.hh | 8 ++-- src/libstore/serve-protocol.cc | 4 +- src/libstore/serve-protocol.hh | 12 +++--- src/libstore/store-api.cc | 2 + src/libstore/store-api.hh | 11 ++++-- src/libstore/worker-protocol-impl.hh | 8 ++-- src/libstore/worker-protocol.cc | 24 ++++++------ src/libstore/worker-protocol.hh | 12 +++--- src/nix-build/nix-build.cc | 1 + src/nix-copy-closure/nix-copy-closure.cc | 1 + 30 files changed, 175 insertions(+), 157 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..210d50b6e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -9,6 +9,7 @@ #undef do_close #include "derivations.hh" +#include "realisation.hh" #include "globals.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 9a2dce806..8e2efc7c3 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const ); } -nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); } -nlohmann::json BuiltPath::toJSON(const Store & store) const +nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..51918f96c 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -11,9 +11,9 @@ struct SingleBuiltPathBuilt { SingleDerivedPathBuilt discardOutputPath() const; - std::string to_string(const Store & store) const; - static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(SingleBuiltPathBuilt); }; @@ -38,8 +38,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { SingleDerivedPath discardOutputPath() const; - static SingleBuiltPath parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; }; static inline ref staticDrv(StorePath drvPath) @@ -56,9 +56,9 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; - std::string to_string(const Store & store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(BuiltPathBuilt); }; @@ -86,7 +86,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..824f94ba1 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -21,7 +21,7 @@ struct AttrDb { std::atomic_bool failed{false}; - const Store & cfg; + const StoreDirConfig & cfg; struct State { @@ -38,7 +38,7 @@ struct AttrDb SymbolTable & symbols; AttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) : cfg(cfg) @@ -322,7 +322,7 @@ struct AttrDb }; static std::shared_ptr makeAttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) { diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index b86ef6b93..27147a5d1 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..20f268a3e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -66,7 +66,6 @@ class Symbol; class PosIdx; struct Pos; class StorePath; -class Store; class EvalState; class XMLWriter; diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c8911d153..9283251ac 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "buildenv.hh" +#include "derivations.hh" #include #include diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0923c2adb..8bebd390d 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "derivations.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh index 079c182b8..360882c02 100644 --- a/src/libstore/common-protocol-impl.hh +++ b/src/libstore/common-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..c14a6cfcd 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -14,40 +14,40 @@ namespace nix { /* protocol-agnostic definitions */ -std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +std::string CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return readString(conn.from); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str) { conn.to << str; } -StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return store.parseStorePath(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { conn.to << store.printStorePath(storePath); } -ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca) { conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { std::string rawInput = readString(conn.from); return Realisation::fromJSON( @@ -56,41 +56,41 @@ Realisation CommonProto::Serialise::read(const Store & store, Commo ); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) { conn.to << realisation.toJSON().dump(); } -DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return DrvOutput::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) { conn.to << drvOutput.to_string(); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parseOpt(readString(conn.from)); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & caOpt) { conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh index f3f28972a..a878e84c9 100644 --- a/src/libstore/common-protocol.hh +++ b/src/libstore/common-protocol.hh @@ -5,7 +5,7 @@ namespace nix { -class Store; +struct StoreDirConfig; struct Source; // items being serialized @@ -48,7 +48,7 @@ struct CommonProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { CommonProto::Serialise::write(store, conn, t); } @@ -57,8 +57,8 @@ struct CommonProto #define DECLARE_COMMON_SERIALISER(T) \ struct CommonProto::Serialise< T > \ { \ - static T read(const Store & store, CommonProto::ReadConn conn); \ - static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \ } template<> diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fecd1c97..239232c8e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,7 +11,7 @@ namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -35,7 +35,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -215,7 +215,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( - const Store & store, + const StoreDirConfig & store, std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { @@ -262,7 +262,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const Store & store, std::istringstream & str, + const StoreDirConfig & store, std::istringstream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -291,7 +291,7 @@ enum struct DerivationATermVersion { }; static DerivedPathMap::ChildNode parseDerivedPathMapNode( - const Store & store, + const StoreDirConfig & store, std::istringstream & str, DerivationATermVersion version) { @@ -338,7 +338,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( Derivation parseDerivation( - const Store & store, std::string && s, std::string_view name, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings) { Derivation drv; @@ -471,7 +471,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } -static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { s += ','; if (node.childMap.empty()) { @@ -512,7 +512,7 @@ static bool hasDynamicDrvDep(const Derivation & drv) } -std::string Derivation::unparse(const Store & store, bool maskOutputs, +std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; @@ -846,7 +846,7 @@ std::map staticOutputHashes(Store & store, const Derivation & } -static DerivationOutput readDerivationOutput(Source & in, const Store & store) +static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) { const auto pathS = readString(in); const auto hashAlgo = readString(in); @@ -863,7 +863,7 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -885,7 +885,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name) { drv.name = name; @@ -913,7 +913,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, } -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) { @@ -1154,7 +1154,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, OutputNameView outputName) const + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -1181,7 +1181,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, OutputNameView outputName, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { @@ -1250,7 +1250,7 @@ DerivationOutput DerivationOutput::fromJSON( } -nlohmann::json Derivation::toJSON(const Store & store) const +nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const { nlohmann::json res = nlohmann::json::object(); @@ -1303,7 +1303,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e7536..219e8e7d7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,7 +17,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /* Abstract syntax of derivations. */ @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -132,17 +132,17 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -304,7 +304,7 @@ struct BasicDerivation * augmented with knowledge of the Store paths they would be written * into. */ - DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -318,6 +318,8 @@ struct BasicDerivation me->name); }; +class Store; + struct Derivation : BasicDerivation { /** @@ -328,7 +330,7 @@ struct Derivation : BasicDerivation /** * Print a derivation. */ - std::string unparse(const Store & store, bool maskOutputs, + std::string unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** @@ -365,9 +367,9 @@ struct Derivation : BasicDerivation Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; static Derivation fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store, * Read a derivation from a file. */ Derivation parseDerivation( - const Store & store, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -493,8 +495,8 @@ extern Sync drvHashes; struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name); +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv); /** * This creates an opaque and almost certainly unique string diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 47d784deb..3105dbc93 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include @@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs) #undef CMP #undef CMP_ONE -nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { return store.printStorePath(path); } @@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const }, raw()); } -std::string DerivedPath::Opaque::to_string(const Store & store) const +std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); } -std::string SingleDerivedPath::Built::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + "^" + output; } -std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string(store) + "!" + output; } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + '^' + outputs.to_string(); } -std::string DerivedPath::Built::to_string_legacy(const Store & store) const +std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string_legacy(store) + "!" + outputs.to_string(); } -std::string SingleDerivedPath::to_string(const Store & store) const +std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string SingleDerivedPath::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const }, this->raw()); } -std::string DerivedPath::to_string_legacy(const Store & store) const +std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const } -DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s) { return {store.parseStorePath(s)}; } @@ -166,7 +167,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse( } static SingleDerivedPath parseWithSingle( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle( } SingleDerivedPath SingleDerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse( } SingleDerivedPath SingleDerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy( } static DerivedPath parseWith( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -240,7 +241,7 @@ static DerivedPath parseWith( } DerivedPath DerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse( } DerivedPath DerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..b12f9734a 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -12,6 +12,9 @@ namespace nix { +struct StoreDirConfig; + +// TODO stop needing this, `toJSON` below should be pure class Store; /** @@ -24,9 +27,9 @@ class Store; struct DerivedPathOpaque { StorePath path; - std::string to_string(const Store & store) const; - static DerivedPathOpaque parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; @@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const Store & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -182,18 +185,18 @@ struct DerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const Store & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh index 4061b0cd6..0cf950a47 100644 --- a/src/libstore/length-prefixed-protocol-helper.hh +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -10,7 +10,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /** * Reusable serialisers for serialization container types in a @@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper; #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ struct LengthPrefixedProtoHelper< Inner, T > \ { \ - static T read(const Store & store, typename Inner::ReadConn conn); \ - static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \ + static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \ private: \ template using S = typename Inner::template Serialise; \ } @@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); template std::vector LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::vector resSet; auto size = readNum(conn.from); @@ -80,7 +80,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -91,7 +91,7 @@ LengthPrefixedProtoHelper>::write( template std::set LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::set resSet; auto size = readNum(conn.from); @@ -104,7 +104,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::set & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -115,7 +115,7 @@ LengthPrefixedProtoHelper>::write( template std::map LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::map resMap; auto size = readNum(conn.from); @@ -130,7 +130,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::map & resMap) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { @@ -142,7 +142,7 @@ LengthPrefixedProtoHelper>::write( template std::tuple LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { return std::tuple { S::read(store, conn)..., @@ -152,7 +152,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::tuple & res) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple & res) { std::apply([&](const Us &... args) { (S::write(store, conn, args), ...); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a5e9426f8..e091683dc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -4,6 +4,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "realisation.hh" #include "nar-info.hh" #include "references.hh" #include "callback.hh" diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1035691c7..9f63fbbb5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "realisation.hh" #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index af6837370..026e37647 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -5,7 +5,7 @@ namespace nix { -std::string StorePathWithOutputs::to_string(const Store & store) const +std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) @@ -85,7 +85,7 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs) { auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..5f76a583a 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,6 +6,8 @@ namespace nix { +struct StoreDirConfig; + /** * This is a deprecated old type just for use by the old CLI, and older * versions of the RPC protocols. In new code don't use it; you want @@ -19,7 +21,7 @@ struct StorePathWithOutputs StorePath path; std::set outputs; - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; DerivedPath toDerivedPath() const; @@ -32,14 +34,14 @@ std::vector toDerivedPaths(const std::vector) std::pair parsePathWithOutputs(std::string_view s); -class Store; - /** * Split a string specifying a derivation and a set of outputs * (/nix/store/hash-foo!out1,out2,...) into the derivation path * and the outputs. */ -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs); + +class Store; StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index a3ce81026..6f3b177ac 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER( template struct ServeProto::Serialise { - static T read(const Store & store, ServeProto::ReadConn conn) + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..e0ac80c4e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -13,7 +13,7 @@ namespace nix { /* protocol-specific definitions */ -BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { BuildResult status; status.status = (BuildResult::Status) readInt(conn.from); @@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise::read(const Store & store, ServeP return status; } -void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +void ServeProto::Serialise::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) { conn.to << status.status diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ba159f6e9..6e9d66e2d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -13,7 +13,7 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -72,8 +72,8 @@ struct ServeProto // See `worker-protocol.hh` for a longer explanation. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -82,7 +82,7 @@ struct ServeProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { ServeProto::Serialise::write(store, conn, t); } @@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) #define DECLARE_SERVE_SERIALISER(T) \ struct ServeProto::Serialise< T > \ { \ - static T read(const Store & store, ServeProto::ReadConn conn); \ - static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a681bb6cf..e44376fa2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,8 @@ #include "crypto.hh" #include "source-accessor.hh" #include "globals.hh" +#include "derived-path.hh" +#include "realisation.hh" #include "derivations.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bee5ec16c..4342445ba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#include "nar-info.hh" -#include "realisation.hh" #include "path.hh" #include "derived-path.hh" #include "hash.hh" @@ -68,8 +66,13 @@ MakeError(SubstituterDisabled, Error); MakeError(InvalidStoreURI, Error); +struct Realisation; +struct RealisedPath; +struct DrvOutput; + struct BasicDerivation; struct Derivation; + struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -811,7 +814,7 @@ void copyStorePath( */ std::map copyPaths( Store & srcStore, Store & dstStore, - const RealisedPath::Set &, + const std::set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -828,7 +831,7 @@ std::map copyPaths( */ void copyClosure( Store & srcStore, Store & dstStore, - const RealisedPath::Set & paths, + const std::set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index c043588d6..026cc37bc 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER( template struct WorkerProto::Serialise { - static T read(const Store & store, WorkerProto::ReadConn conn) + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..4edab7894 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -14,7 +14,7 @@ namespace nix { /* protocol-specific definitions */ -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto temp = readNum(conn.from); switch (temp) { @@ -29,7 +29,7 @@ std::optional WorkerProto::Serialise>::r } } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) conn.to << (uint8_t)0; @@ -48,7 +48,7 @@ void WorkerProto::Serialise>::write(const Store & sto } -DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); if (GET_PROTOCOL_MINOR(conn.version) >= 30) { @@ -58,7 +58,7 @@ DerivedPath WorkerProto::Serialise::read(const Store & store, Worke } } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) { if (GET_PROTOCOL_MINOR(conn.version) >= 30) { conn.to << req.to_string_legacy(store); @@ -82,7 +82,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); auto br = WorkerProto::Serialise::read(store, conn); @@ -92,14 +92,14 @@ KeyedBuildResult WorkerProto::Serialise::read(const Store & st }; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) { WorkerProto::write(store, conn, res.path); WorkerProto::write(store, conn, static_cast(res)); } -BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); @@ -121,7 +121,7 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke return res; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) { conn.to << res.status @@ -142,7 +142,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); return ValidPathInfo { @@ -151,14 +151,14 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R }; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo) { WorkerProto::write(store, conn, pathInfo.path); WorkerProto::write(store, conn, static_cast(pathInfo)); } -UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); @@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St return info; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..9b02aa2b5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -24,7 +24,7 @@ namespace nix { #define STDERR_RESULT 0x52534c54 -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -100,8 +100,8 @@ struct WorkerProto // This makes for a quicker debug cycle, as desired. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -110,7 +110,7 @@ struct WorkerProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { WorkerProto::Serialise::write(store, conn, t); } @@ -197,8 +197,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) #define DECLARE_WORKER_SERIALISER(T) \ struct WorkerProto::Serialise< T > \ { \ - static T read(const Store & store, WorkerProto::ReadConn conn); \ - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 60bc08146..c46095a14 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -13,6 +13,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "realisation.hh" #include "derivations.hh" #include "util.hh" #include "shared.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 7f2bb93b6..b64af758f 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,4 +1,5 @@ #include "shared.hh" +#include "realisation.hh" #include "store-api.hh" #include "legacy.hh" From e4cbdd26e0e6a2a5907dff8e60c3645f7d94423a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:13:52 +0100 Subject: [PATCH 04/54] Add TODO comment for include try/catch --- src/libutil/config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 17380b6d8..ab873b4a8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -125,7 +125,9 @@ static void applyConfigInner(const std::string & contents, const std::string & p try { std::string includedContents = readFile(path); applyConfigInner(includedContents, p, parsedContents); - } catch (SysError &) { } + } catch (SysError &) { + // TODO: Do we actually want to ignore this? Or is it better to fail? + } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } From d6898cd58b1a685404ba6878c317e60be9473a9a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:14:05 +0100 Subject: [PATCH 05/54] Move applyConfigFile to lambda inside libstore --- src/libstore/globals.cc | 11 +++++++++-- src/libutil/config.cc | 8 -------- src/libutil/config.hh | 6 ------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..0aecd2b6a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -111,7 +111,14 @@ Settings::Settings() void loadConfFile() { - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + auto applyConfigFile = [&](const Path & path) { + try { + std::string contents = readFile(path); + globalConfig.applyConfig(contents, path); + } catch (SysError &) { } + }; + + applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ @@ -119,7 +126,7 @@ void loadConfFile() auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - globalConfig.applyConfigFile(*file); + applyConfigFile(*file); } auto nixConfEnv = getEnv("NIX_CONFIG"); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index ab873b4a8..ad16c86bd 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -165,14 +165,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - std::string contents = readFile(path); - applyConfig(contents, path); - } catch (SysError &) { } -} - void Config::resetOverridden() { for (auto & s : _settings) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..d49eb602d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -82,12 +82,6 @@ public: */ void applyConfig(const std::string & contents, const std::string & path = ""); - /** - * Applies a nix configuration file - * - path: the location of the config file to apply - */ - void applyConfigFile(const Path & path); - /** * Resets the `overridden` flag of all Settings */ From 4e790efade0c3073292ff73be44351f29badd935 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Nov 2023 13:38:52 +0100 Subject: [PATCH 06/54] Use boost::container::small_vector in place of VLAs --- boehmgc-traceable_allocator-public.diff | 12 +++++++ flake.nix | 3 ++ src/libexpr/eval.cc | 13 +++++--- src/libexpr/gc-small-vector.hh | 42 +++++++++++++++++++++++++ src/libexpr/primops.cc | 9 +++--- 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 boehmgc-traceable_allocator-public.diff create mode 100644 src/libexpr/gc-small-vector.hh diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff new file mode 100644 index 000000000..903c707a6 --- /dev/null +++ b/boehmgc-traceable_allocator-public.diff @@ -0,0 +1,12 @@ +diff --git a/include/gc_allocator.h b/include/gc_allocator.h +index 597c7f13..587286be 100644 +--- a/include/gc_allocator.h ++++ b/include/gc_allocator.h +@@ -312,6 +312,7 @@ public: + + template<> + class traceable_allocator { ++public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; diff --git a/flake.nix b/flake.nix index 9030a74f7..570a099ab 100644 --- a/flake.nix +++ b/flake.nix @@ -230,6 +230,9 @@ }).overrideAttrs(o: { patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff ]; }) ) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46a49c891..90f04d40a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -16,6 +16,7 @@ #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" +#include "gc-small-vector.hh" #include #include @@ -31,6 +32,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -1709,7 +1711,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1772,11 +1774,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // 4: about 60 // 5: under 10 // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. - Value * vArgs[args.size()]; + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -2015,8 +2017,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000..7f4f08fc7 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a8d44d8b7..54a5da817 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4,6 +4,7 @@ #include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" +#include "gc-small-vector.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" @@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; + SmallValueVector res(args[1]->listSize()); size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; + SmallValueVector vs(args[1]->listSize()); size_t k = 0; bool same = true; @@ -3454,7 +3454,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From b7982372d234b1fd15bab01d09093471c1870bb4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 15:05:21 +0100 Subject: [PATCH 07/54] Compile hand-written release notes with changelog-d --- .gitignore | 1 + doc/manual/local.mk | 5 +++- doc/manual/rl-next/config | 2 ++ doc/manual/src/contributing/hacking.md | 38 ++++++++++++++++++++++++++ flake.nix | 11 +++++++- 5 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 doc/manual/rl-next/config diff --git a/.gitignore b/.gitignore index 04d96ca2c..38e7e2b5a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ perl/Makefile.config /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md /doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh diff --git a/doc/manual/local.mk b/doc/manual/local.mk index db3daf252..74a4103b3 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -144,6 +144,9 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ +$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* + $(trace-gen) changelog-d doc/manual/rl-next > $@ + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html @@ -177,7 +180,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 000000000..b3c2e868f --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index fe08ceb94..855900d7a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -220,3 +220,41 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +synopsis: Basically a title +issues: #1234 +prs: #1238 +description: { + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ + +} +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. +Non-releases build the release notes on the fly. diff --git a/flake.nix b/flake.nix index 9030a74f7..ca37948a7 100644 --- a/flake.nix +++ b/flake.nix @@ -173,6 +173,10 @@ "--enable-internal-api-docs" ]; + # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - + # vastly improves output closure, and adds shell completions + changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + nativeBuildDeps = [ buildPackages.bison @@ -190,7 +194,10 @@ buildPackages.jq # Also for custom mdBook preprocessor. buildPackages.openssh # only needed for tests (ssh-keygen) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease) changelog-d + ; buildDeps = [ curl @@ -727,6 +734,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools + # We want changelog-d in the shell even if it's an official release + ++ lib.optional officialRelease changelog-d ; buildInputs = buildDeps ++ propagatedDeps From b26038c517ed10feae751ad6733244c00b715d34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:08:34 +0100 Subject: [PATCH 08/54] doc: Rename 2X.XX to "Upcoming release", and only generate if applicable --- .gitignore | 1 + doc/manual/local.mk | 13 +++++++++++-- doc/manual/src/SUMMARY.md.in | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 38e7e2b5a..5c359d7a0 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ perl/Makefile.config /doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 74a4103b3..265a4649d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -92,7 +92,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -144,9 +144,18 @@ $(d)/language.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ -$(d)/src/release-notes/rl-next.md: $(d)/rl-next/* +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* $(trace-gen) changelog-d doc/manual/rl-next > $@ +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8dc464abd..8e7b4eeab 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -114,7 +114,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +{{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) From 2a538c571b13877fa426f2cff2749cf17d140216 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 16:49:11 +0100 Subject: [PATCH 09/54] Add scripts/release-notes --- maintainers/release-process.md | 45 ++++------ scripts/release-notes | 148 +++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 27 deletions(-) create mode 100755 scripts/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index d85266b81..e542d8be5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,34 +24,23 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. -* Move the contents of `doc/manual/src/release-notes/rl-next.md` - (except the first line) to - `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is - the contents of `.version` *without* the patch level, e.g. `2.12` - rather than `2.12.0`). - -* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like - - ``` - # Release 2.12 (2022-12-06) - ``` - -* Proof-read / edit / rearrange the release notes. Breaking changes - and highlights should go to the top. - -* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` - (*not* `SUMMARY.md`), e.g. - - ``` - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - ``` - -* Run +* Compile the release notes by running ```console $ git checkout -b release-notes - $ git add doc/manual/src/release-notes/rl-$VERSION.md - $ git commit -a -m 'Release notes' + $ VERSION=X.YY ./scripts/release-notes + ``` + + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. + + A commit is created. + +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. + +* Push. + + ```console $ git push --set-upstream $REMOTE release-notes ``` @@ -67,15 +56,17 @@ release: $ git checkout -b $VERSION-maintenance ``` -* Mark the release as stable: +* Mark the release as official: ```console - $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix ``` This removes the link to `rl-next.md` from the manual and sets `officialRelease = true` in `flake.nix`. +* Commit + * Push the release branch: ```console diff --git a/scripts/release-notes b/scripts/release-notes new file mode 100755 index 000000000..e5ee39d11 --- /dev/null +++ b/scripts/release-notes @@ -0,0 +1,148 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small +# ^^^^^^^ +# Only used for bash. shell.nix goes to the flake. + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +case "$VERSION" in + # FIXME: accepts "." without any real digits + [[:digit:]]*.[[:digit:]]*) + ;; + *) + die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" + ;; +esac + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename +title="Release $VERSION ($DATE)" + +( + # TODO add minor number, and append? + echo "# $title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' +) > $file + +log "Wrote $file" + +NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + +# find the marker line, insert new link after it +escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" +logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $VERSION" + +report_done From b1ea30f21d24df9afc4eb1635eee9a080e4f81f3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:13:59 +0100 Subject: [PATCH 10/54] scripts/release-notes: Support patch releases This also fixes the broken case statement, which has globs, not regexes. --- maintainers/release-process.md | 24 +++++++++++++ scripts/release-notes | 65 ++++++++++++++++++++++++---------- 2 files changed, 71 insertions(+), 18 deletions(-) diff --git a/maintainers/release-process.md b/maintainers/release-process.md index e542d8be5..b1259a9bd 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -150,6 +150,30 @@ release: ## Creating a point release +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./scripts/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + * Wait for the desired evaluation of the maintenance jobset to finish building. diff --git a/scripts/release-notes b/scripts/release-notes index e5ee39d11..43f20a547 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,14 +89,40 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -case "$VERSION" in - # FIXME: accepts "." without any real digits - [[:digit:]]*.[[:digit:]]*) - ;; - *) - die "VERSION must be MAJOR.MINOR, where each is a number, e.g. 2.20 (VERSION was set to $VERSION)" - ;; -esac +# mutate/initialize: +# VERSION: MAJOR.MINOR +# FULL_VERSION: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + FULL_VERSION="$VERSION.0" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + FULL_VERSION="$VERSION" + VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +log "VERSION=$VERSION" +log "FULL_VERSION=$FULL_VERSION" +log "IS_PATCH=$IS_PATCH" + +basename=rl-$VERSION.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi # --- DEFAULTS --- @@ -106,7 +132,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [[:digit:]]*-[[:digit:]]*-[[:digit:]]*) + [0-9]*-[0-9]*-[0-9]*) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" @@ -115,25 +141,28 @@ esac # --- DO THE WORK --- -basename=rl-$VERSION.md -file=doc/manual/src/release-notes/$basename +# menu title="Release $VERSION ($DATE)" +# section on page +section_title="Release $FULL_VERSION ($DATE)" ( # TODO add minor number, and append? - echo "# $title" + echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' -) > $file +) | tee -a $file log "Wrote $file" -NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" -# find the marker line, insert new link after it -escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" -logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi for f in doc/manual/rl-next/*.md; do if [[ config != "$(basename $f)" ]]; then From 7c4ee5c8135fae65602791f0b89d0dbae7e94f3e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 18:36:50 +0100 Subject: [PATCH 11/54] scripts/release-notes: Avoid mutating variables --- scripts/release-notes | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/scripts/release-notes b/scripts/release-notes index 43f20a547..2e1be64a4 100755 --- a/scripts/release-notes +++ b/scripts/release-notes @@ -89,33 +89,35 @@ if [[ ! -n "${VERSION:-}" ]]; then exit 1 fi -# mutate/initialize: -# VERSION: MAJOR.MINOR -# FULL_VERSION: MAJOR.MINOR.PATCH -# IS_PATCH: true if this is a patch release; append instead of create +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is minor' IS_PATCH=false - FULL_VERSION="$VERSION.0" + version_full="$VERSION.0" + version_major_minor="$VERSION" elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then log 'is minor (.0)' IS_PATCH=false - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.0$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then log 'is patch' IS_PATCH=true - FULL_VERSION="$VERSION" - VERSION="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" else die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" fi -log "VERSION=$VERSION" -log "FULL_VERSION=$FULL_VERSION" +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" log "IS_PATCH=$IS_PATCH" -basename=rl-$VERSION.md +basename=rl-${version_major_minor}.md file=doc/manual/src/release-notes/$basename if ! $IS_PATCH; then @@ -142,9 +144,9 @@ esac # --- DO THE WORK --- # menu -title="Release $VERSION ($DATE)" +title="Release $version_major_minor ($DATE)" # section on page -section_title="Release $FULL_VERSION ($DATE)" +section_title="Release $version_full ($DATE)" ( # TODO add minor number, and append? @@ -172,6 +174,6 @@ done logcmd git add $file doc/manual/src/SUMMARY.md.in logcmd git status -logcmd git commit -m "release notes: $VERSION" +logcmd git commit -m "release notes: $version_full" report_done From 6971c4adc06d574cbe1e9ab6da19814e11e2ba6c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 19 Nov 2023 19:08:42 +0100 Subject: [PATCH 12/54] maintainers/release-notes <- scripts/release-notes --- {scripts => maintainers}/release-notes | 0 maintainers/release-process.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename {scripts => maintainers}/release-notes (100%) diff --git a/scripts/release-notes b/maintainers/release-notes similarity index 100% rename from scripts/release-notes rename to maintainers/release-notes diff --git a/maintainers/release-process.md b/maintainers/release-process.md index b1259a9bd..db8b064a5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -28,7 +28,7 @@ release: ```console $ git checkout -b release-notes - $ VERSION=X.YY ./scripts/release-notes + $ VERSION=X.YY ./maintainers/release-notes ``` where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. @@ -165,7 +165,7 @@ release: * Update release notes. ```console - $ ./scripts/release-notes + $ ./maintainers/release-notes ``` * Push. From 857f9168f7b48aa491052f24fb571c21398f9826 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:27:54 +0100 Subject: [PATCH 13/54] Migrate rl-next.md to doc/manual/rl-next directory --- doc/manual/rl-next/mounted-ssh-store.md | 9 +++++++++ doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++++++++ doc/manual/src/release-notes/rl-next.md | 6 ------ 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 doc/manual/rl-next/mounted-ssh-store.md create mode 100644 doc/manual/rl-next/nix-env-json-drv-path.md delete mode 100644 doc/manual/src/release-notes/rl-next.md diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md new file mode 100644 index 000000000..39fac5283 --- /dev/null +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -0,0 +1,9 @@ +synopsis: Mounted SSH Store +issues: #7890 +prs: #7912 +description: { + +Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). +This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md new file mode 100644 index 000000000..fbe2b67d8 --- /dev/null +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -0,0 +1,9 @@ +synopsis: Fix `nix-env --query --drv-path --json` +prs: #9257 +description: { + +Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +} + + diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 0e3d8b462..000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release X.Y (202?-??-??) - -- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. - -- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. From d2f5e263e3c095dfe9d874387665b88c4bfff6f1 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 6 Feb 2023 16:36:57 +0100 Subject: [PATCH 14/54] Switch from std::regex to boost::regex --- flake.nix | 24 +++++++++++++++++------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 +++++++++++++++++++++++------------ 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/flake.nix b/flake.nix index 9030a74f7..1f7e7b2af 100644 --- a/flake.nix +++ b/flake.nix @@ -154,7 +154,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" + "--with-boost=${boost-nix}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -202,7 +202,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost + boost-nix lowdown-nix libsodium ] @@ -423,14 +423,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost ]; + disallowedReferences = [ boost-nix ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -440,9 +440,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -470,9 +470,13 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ + -change ${boost-nix}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib + install_name_tool \ + -change ${boost-nix}/lib/libboost_regex.dylib \ + $out/lib/libboost_regex.dylib \ + $out/lib/libnixexpr.dylib ''} ''; @@ -495,6 +499,12 @@ meta.mainProgram = "nix"; }); + boost-nix = final.boost.override { + # enableIcu arg is not yet supported + # but will be with next nixpkgs update + enableIcu = false; + }; + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf9490..946059339 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -pthread +libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ba735b435..0c34cb6e8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,6 +16,7 @@ #include "primops.hh" #include +#include #include #include @@ -24,7 +25,6 @@ #include #include -#include #include #include @@ -3885,19 +3885,30 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); +// regex aliases, switch between boost and std +using regex = boost::regex; +using regex_error = boost::regex_error; +using cmatch = boost::cmatch; +using cregex_iterator = boost::cregex_iterator; +namespace regex_constants = boost::regex_constants; +// overloaded function alias +constexpr auto regex_match = [] (auto &&...args) { + return boost::regex_match(std::forward(args)...); + }; + struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - std::regex get(std::string_view re) + regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; } }; @@ -3917,8 +3928,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - std::cmatch match; - if (!std::regex_match(str.begin(), str.end(), match, regex)) { + cmatch match; + if (!regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3933,8 +3944,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -3997,8 +4008,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = std::cregex_iterator(str.begin(), str.end(), regex); - auto end = std::cregex_iterator(); + auto begin = cregex_iterator(str.begin(), str.end(), regex); + auto end = cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4037,8 +4048,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (std::regex_error & e) { - if (e.code() == std::regex_constants::error_space) { + } catch (regex_error & e) { + if (e.code() == regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From f25c06d7a3289343887d09761c82356a3b6b441b Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sun, 26 Nov 2023 15:46:09 +0100 Subject: [PATCH 15/54] docs: Fix broken link Link target definitions need to be in a separate paragraph to be collected. Fixup for https://github.com/NixOS/nix/commit/217d863f7a251a4d8a08ff3294944b45146c61c9 --- doc/manual/src/language/values.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 0bb656746..aea68a441 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -116,6 +116,7 @@ [store path]: ../glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. From d63f72197cec6ff95d9ffc83aa8076acd86a3fd1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 15:48:41 +0100 Subject: [PATCH 16/54] Don't run changelog-d in the build This way we lose the preview of release notes on master, as well as on https://nixos.org/manual/nix/unstable/release-notes/rl-next but we can come back to this. --- doc/manual/local.mk | 8 +++++++- doc/manual/src/contributing/hacking.md | 2 +- flake.nix | 9 ++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 265a4649d..f22dfa69e 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -146,7 +146,13 @@ $(d)/language.json: $(bindir)/nix # Generate "Upcoming release" notes (or clear it and remove from menu) $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* - $(trace-gen) changelog-d doc/manual/rl-next > $@ + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md $(trace-gen) true diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 855900d7a..3291d5a20 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -257,4 +257,4 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master ### Build process Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. -Non-releases build the release notes on the fly. +Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. diff --git a/flake.nix b/flake.nix index ca37948a7..db913f062 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,9 @@ officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -196,7 +199,7 @@ ] ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease) changelog-d + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d ; buildDeps = @@ -734,8 +737,8 @@ ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if it's an official release - ++ lib.optional officialRelease changelog-d + # We want changelog-d in the shell even if the current build doesn't need it + ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d ; buildInputs = buildDeps ++ propagatedDeps From c5d49ec7ab7b9fb33f0336a909ac837e208be807 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:18:27 +0100 Subject: [PATCH 17/54] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/decdf666c833a325cb4417041a90681499e06a41' (2023-11-18) → 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-23) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 1b3c0b6d5..f120d3b5f 100644 --- a/flake.lock +++ b/flake.lock @@ -50,11 +50,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700342017, - "narHash": "sha256-HaibwlWH5LuqsaibW3sIVjZQtEM/jWtOHX4Nk93abGE=", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "decdf666c833a325cb4417041a90681499e06a41", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { From e7e21aa0c839460a62456fa44f31339c187077ff Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 24 Nov 2023 16:27:10 +0100 Subject: [PATCH 18/54] flake.nix: Use top level changelog-d It is about 2 MB now, as only it and libffi (tiny) are new in the build or shell closures. --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index db913f062..a239226c0 100644 --- a/flake.nix +++ b/flake.nix @@ -176,9 +176,7 @@ "--enable-internal-api-docs" ]; - # TODO: after backport of https://github.com/NixOS/nixpkgs/pull/268487, remove `haskellPackages.` - - # vastly improves output closure, and adds shell completions - changelog-d = pkgs.buildPackages.haskellPackages.changelog-d; + changelog-d = pkgs.buildPackages.changelog-d; nativeBuildDeps = [ From 2b7016cc56d12e67de9f1f25b18311866a26a5fe Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 08:33:03 +0100 Subject: [PATCH 19/54] add path based redirects up to now, those were managed outside of this repo, which as unsurprisingly a real hassle to deal with if one wanted to prevent URLs from breaking when moving pages around. this change removes a large part of the friction involved in moving content in the Nix manual. possible next steps for further automation: - check for content that moved and warn if it's not reachable from links that were valid prior to a change - create redirect rules automatically based on this information --- doc/manual/_redirects | 30 ++++++++++++++++++++++++++++++ doc/manual/redirects.js | 8 +++++--- 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 doc/manual/_redirects diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 000000000..4ea289d86 --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,30 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - always append new redirects to the end of the file +# - redirects that should have been there but are missing can be inserted where they belong + +/expressions/expression-language /language/ 301! +/expressions/language-values /language/values 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/* /language/:splat 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! + +/package-management/channels* /command-ref/nix-channel 301! + +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d1b10109d..3b507adf3 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix From f56401a114cb5504c3d74b893ce270ed28fd03e3 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Sat, 25 Nov 2023 11:26:57 +0800 Subject: [PATCH 20/54] `nix flake update` add deprecation warnings. This builds on #8817, to add additional UX help for people with existing muscle memory (or shell history) with --update-input and tries to gently guide them towards the newly evolved CLI UI. Co-authored-by: Cole Helbling --- src/libcmd/installables.cc | 24 ++++++++++++++++++++++++ src/nix/flake.cc | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 1c6103020..68287b445 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -47,6 +47,16 @@ MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; + addFlag({ + .longName = "recreate-lock-file", + .description = "Recreate the flake's lock file from scratch.", + .category = category, + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} + }); + addFlag({ .longName = "no-update-lock-file", .description = "Do not allow any updates to the flake's lock file.", @@ -79,6 +89,20 @@ MixFlakeOptions::MixFlakeOptions() .handler = {&lockFlags.commitLockFile, true} }); + addFlag({ + .longName = "update-input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .category = category, + .labels = {"input-path"}, + .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); + lockFlags.inputUpdates.insert(flake::parseInputPath(s)); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + addFlag({ .longName = "override-input", .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 38938f09e..e0c67fdfa 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -89,7 +89,13 @@ public: .label="inputs", .optional=true, .handler={[&](std::string inputToUpdate){ - auto inputPath = flake::parseInputPath(inputToUpdate); + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } if (lockFlags.inputUpdates.contains(inputPath)) warn("Input '%s' was specified multiple times. You may have done this by accident."); lockFlags.inputUpdates.insert(inputPath); From 75134b7513eb781074969fc8d6d865cc95063444 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 27 Nov 2023 08:56:24 +0000 Subject: [PATCH 21/54] libexpr: add missing dependency on 'flake/call-flake.nix.gen.hh' Without the change build for `eval.o` fails occasionally as: $ make src/libexpr/eval.o GEN Makefile.config GEN src/libexpr/primops/derivation.nix.gen.hh GEN src/libexpr/fetchurl.nix.gen.hh GEN src/libexpr/parser-tab.cc GEN src/libexpr/lexer-tab.cc src/libexpr/lexer.l:314: warning, -s option given but default rule can be matched CXX src/libexpr/eval.o src/libexpr/eval.cc:519:18: fatal error: flake/call-flake.nix.gen.hh: No such file or directory 519 | #include "flake/call-flake.nix.gen.hh" | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ compilation terminated. make: *** [mk/patterns.mk:3: src/libexpr/eval.o] Error 1 Noticed in https://github.com/NixOS/nixpkgs/pull/269439 --- src/libexpr/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ed7bf9490..637f998b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -47,6 +47,6 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From e986d20bedfc054663632255388bbd33fec99114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:25:18 +0100 Subject: [PATCH 22/54] Remove an obsolete comment --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1f7e7b2af..887ef93b4 100644 --- a/flake.nix +++ b/flake.nix @@ -500,8 +500,6 @@ }); boost-nix = final.boost.override { - # enableIcu arg is not yet supported - # but will be with next nixpkgs update enableIcu = false; }; From 384ffb4443fd47d04f36c6bcc6ebf476274673ab Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 27 Nov 2023 09:18:56 +0100 Subject: [PATCH 23/54] add deprecation warnings in documentation this is hacky, but can serve as a stopgap until we can do it programmatically. --- src/libcmd/installables.cc | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 68287b445..6e670efea 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -49,7 +49,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .handler = {[&]() { lockFlags.recreateLockFile = true; @@ -73,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -91,7 +102,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { From f7bfec2806708573798d610cda101f27a24d9218 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:18:29 +0100 Subject: [PATCH 24/54] maintainers/release-notes: Improve DATE check Co-authored-by: Valentin Gagarin --- maintainers/release-notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/release-notes b/maintainers/release-notes index 2e1be64a4..34cd85a56 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -134,7 +134,7 @@ if [[ ! -n "${DATE:-}" ]]; then fi case "$DATE" in - [0-9]*-[0-9]*-[0-9]*) + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" From 68c48756fece5aee77f9b44607afa9248d75e67c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 27 Nov 2023 15:50:45 +0100 Subject: [PATCH 25/54] libexpr/local.mk: Make eval compile deps regular Dependency is now entirely through the eval.cc rule. All gen.hh deps are now there. --- src/libexpr/local.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 637f998b6..b37fe6f1d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -45,8 +45,6 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh - -$(d)/eval.o: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = From 20cd5eb2b3668f5b95b6f020e1c258011c18ea33 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Mon, 27 Nov 2023 19:12:15 +0100 Subject: [PATCH 26/54] nix repl: Only hide the progress bar while waiting for user input In commit 0d2163c6dcf03463fa91ec6d0d96c928ad907366, the progress bar was hidden in nix repl because of a regression that caused it to interfere with user input. Several users like(d) seeing the progress bar in the repl during builds. Only hiding it while waiting for user input gives us the best of both worlds, so do just that. --- src/libcmd/repl.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index bf5643a5c..0986296ad 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -43,7 +43,6 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" #include "print.hh" #if HAVE_BOEHMGC @@ -262,13 +261,11 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); - std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { @@ -278,6 +275,7 @@ void NixRepl::mainLoop() logger->cout(""); break; } + logger->resume(); try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { From f300e11b056dea414d7d77bbc6e5a7dc5d9ddd41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Nov 2023 19:41:30 +0100 Subject: [PATCH 27/54] Rename `nix show-config` to `nix config show` Part of #7672 --- doc/manual/local.mk | 4 ++-- doc/manual/rl-next/nix-config-show.md | 8 +++++++ src/nix/{show-config.cc => config.cc} | 27 ++++++++++++++++++++--- src/nix/main.cc | 1 + tests/functional/config.sh | 12 +++++----- tests/functional/experimental-features.sh | 8 +++---- 6 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 doc/manual/rl-next/nix-config-show.md rename src/nix/{show-config.cc => config.cc} (67%) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index f22dfa69e..d568681d4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -24,7 +24,7 @@ man-pages += $(foreach subcommand, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -111,7 +111,7 @@ $(d)/nix.json: $(bindir)/nix @mv $@.tmp $@ $(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ $(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md new file mode 100644 index 000000000..08ad207cb --- /dev/null +++ b/doc/manual/rl-next/nix-config-show.md @@ -0,0 +1,8 @@ +synopsis: `nix config show` +issues: #7672 +prs: #9477 +description: { + +`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. + +} diff --git a/src/nix/show-config.cc b/src/nix/config.cc similarity index 67% rename from src/nix/show-config.cc rename to src/nix/config.cc index 3530584f9..5b280d11d 100644 --- a/src/nix/show-config.cc +++ b/src/nix/config.cc @@ -7,11 +7,31 @@ using namespace nix; -struct CmdShowConfig : Command, MixJSON +struct CmdConfig : virtual NixMultiCommand +{ + CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix config' requires a sub-command."); + command->second->run(); + } +}; + +struct CmdConfigShow : Command, MixJSON { std::optional name; - CmdShowConfig() { + CmdConfigShow() { expectArgs({ .label = {"name"}, .optional = true, @@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto rShowConfig = registerCommand("show-config"); +static auto rCmdConfig = registerCommand("config"); +static auto rShowConfig = registerCommand2({"config", "show"}); diff --git a/src/nix/main.cc b/src/nix/main.cc index 73641f6d2..2a6c2f478 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -134,6 +134,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, {"show-derivation", {"derivation", "show"}}, + {"show-config", {"config", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 0780c55d0..324fe95bd 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -40,20 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] # flakes implies fetch-tree [[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value -val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) -val2=$(nix show-config warn-dirty) +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) [[ $val == $val2 ]] diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 607bf0a8e..9ee4a53d4 100644 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,7 +31,7 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr @@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr From 6d1605818c12461edc9f4ee17a7929fdc8fe916c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Nov 2023 19:46:05 +0100 Subject: [PATCH 28/54] Rename `nix doctor` to `nix config check` Fix #7672 --- doc/manual/src/contributing/cli-guideline.md | 2 +- src/nix/{doctor.cc => config-check.cc} | 4 ++-- src/nix/main.cc | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) rename src/nix/{doctor.cc => config-check.cc} (97%) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..8dbac45b0 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -87,7 +87,7 @@ impacted the most by bad user experience. and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. - Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + Examples of such commands: `nix edit`, `nix eval`, ... - **Utility and scripting commands** diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc similarity index 97% rename from src/nix/doctor.cc rename to src/nix/config-check.cc index 59f9e3e5d..410feca2f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/config-check.cc @@ -38,7 +38,7 @@ void checkInfo(const std::string & msg) { } -struct CmdDoctor : StoreCommand +struct CmdConfigCheck : StoreCommand { bool success = true; @@ -152,4 +152,4 @@ struct CmdDoctor : StoreCommand } }; -static auto rCmdDoctor = registerCommand("doctor"); +static auto rCmdConfigCheck = registerCommand2({ "config", "check" }); diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..d715d0400 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -139,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, {"verify", {"store", "verify"}}, + {"doctor", {"config", "check"}}, }; bool aliasUsed = false; From 52e0911302b20336c1600b60a98894423e110d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 25 Nov 2023 00:33:21 -0500 Subject: [PATCH 29/54] Use `buildprefix` in a few more places `installcheck` doesn't yet work, but the rest of the build can now happen mostly inside a separate build directory. Progress on #9342 Co-authored-by: Valentin Gagarin --- Makefile | 6 ++++-- doc/manual/src/contributing/hacking.md | 25 +++++++++++++++++++++++++ mk/build-dir.mk | 10 ++++++++++ mk/install-dirs.mk | 11 +++++++++++ mk/lib.mk | 25 +++---------------------- mk/templates.mk | 8 ++++---- src/libcmd/local.mk | 2 +- src/libexpr/local.mk | 4 ++-- src/libmain/local.mk | 2 +- src/libstore/local.mk | 2 +- tests/functional/local.mk | 4 ++-- 11 files changed, 64 insertions(+), 35 deletions(-) create mode 100644 mk/build-dir.mk create mode 100644 mk/install-dirs.mk diff --git a/Makefile b/Makefile index 4f4ac0c6e..77974074c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ --include Makefile.config -clean-files += Makefile.config +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config ifeq ($(ENABLE_BUILD), yes) makefiles = \ diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 3291d5a20..0a95334f7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: + +1. Create a directory for the build, e.g. + + ```bash + mkdir build + ``` + +2. Run the configure script from that directory, e.g. + + ```bash + cd build + ../configure + ``` + +3. Run make from the source directory, but with the build directory specified, e.g. + + ```bash + make builddir=build + ``` + ## System type Nix uses a string with he following format to identify the *system type* or *platform* it runs on: diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000..02f4cae60 --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000..732b0d6fc --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index 49abe9862..3d503364f 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -43,27 +43,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -112,7 +93,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbf..866bdc17f 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 541a7d2ba..afd35af08 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 18d1bc95c..c07a18bb5 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) @@ -47,4 +47,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh -src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27..5c7061863 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 0be0bf310..68ccdc409 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -59,7 +59,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 8d584142a..10b399d75 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -140,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes) endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ - $(d)/test-libstoreconsumer/test-libstoreconsumer + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ - $(d)/plugins/libplugintest.$(SO_EXT) + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) From 02bd821f2e71372d31bbe6700bd68086cc2ee70a Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 29 Nov 2023 19:26:39 -0600 Subject: [PATCH 30/54] fix: `nlohmann::adl_serializer` for `std::optional` (#9147) This allows templates such as `NLOHMANN_DEFINE_TYPE_*` templates and other generators with things like `std::vector>`. Co-authored-by: John Ericson --- src/libutil/json-utils.hh | 19 ++++++++--- src/libutil/tests/json-utils.cc | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 src/libutil/tests/json-utils.cc diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 77c63595c..06dd80cf7 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -78,20 +78,29 @@ namespace nlohmann { */ template struct adl_serializer> { - static std::optional from_json(const json & json) { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); - return json.is_null() + t = json.is_null() ? std::nullopt - : std::optional { adl_serializer::from_json(json) }; + : std::make_optional(json.template get()); } - static void to_json(json & json, std::optional t) { + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); if (t) - adl_serializer::to_json(json, *t); + json = *t; else json = nullptr; } diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc new file mode 100644 index 000000000..f0ce15c93 --- /dev/null +++ b/src/libutil/tests/json-utils.cc @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +} /* namespace nix */ From a7115a47ef0d83ea81b494f6bc5b11d8286e0672 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 Nov 2023 21:03:56 -0500 Subject: [PATCH 31/54] Improve ACL clearing support (fixing FreeBSD build) The problem was that f880469173061a07f0b2a24734932c5a9ad633c6 forgot that the `#include ` was guarded by an `#ifdef __linux__`. However, the build failure was only on FreeBSD --- turns out other platforms have this header too! The fix therefore uses a new configure check so we properly clear ACLs on more platforms. --- configure.ac | 2 ++ src/libstore/local-store.cc | 1 - src/libstore/posix-fs-canonicalise.cc | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 71e93feaa..f8b937eb5 100644 --- a/configure.ac +++ b/configure.ac @@ -282,6 +282,8 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS[sys/xattr.h] # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4ff75f528..9ed061b01 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -34,7 +34,6 @@ #include #include #include -#include #endif #ifdef __CYGWIN__ diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index cc3ab0b74..f38fa8369 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,4 +1,6 @@ -#include +#if HAVE_SYS_XATTR_H +# include +#endif #include "posix-fs-canonicalise.hh" #include "file-system.hh" @@ -76,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#if __linux__ +#ifdef HAVE_SYS_XATTR_H /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); From d536c57e878a04f795c1ef8ee3232a47035da2cf Mon Sep 17 00:00:00 2001 From: Federico Pellegrin Date: Tue, 17 Aug 2021 04:26:41 +0200 Subject: [PATCH 32/54] Docs build: depend on locally built nix executable and not installed one Previously many of the documentation targets were depending on `$(bindir)/nix` which is the installed version. This meant that its install rules would be triggered (which in chain would also trigger the install of libraries, as reported in #5140). Therefore a build of the documentation without an installation would not be possible (which apart from doing unwanted operations it may also generate permission problems for example). The fix makes the rules depend on `$(nix_PATH)` instead, which is the executable in the build tree. --- Makefile | 11 ++++++++--- doc/manual/local.mk | 35 ++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 4f4ac0c6e..957920686 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,7 @@ makefiles = \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - doc/internal-api/local.mk + misc/upstart/local.mk endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) @@ -55,4 +53,11 @@ endif include mk/lib.mk +# Must be included after `mk/lib.mk` so rules refer to variables defined +# by the library. Rules are not "lazy" like variables, unfortunately. +ifeq ($(ENABLE_BUILD), yes) +$(eval $(call include-sub-makefile, doc/manual/local.mk)) +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +endif + GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d568681d4..fa9db9f02 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,10 @@ ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) + MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ $(call rwildcard, $(d)/src, */*.md) @@ -32,7 +37,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -96,52 +101,52 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ # Generate "Upcoming release" notes (or clear it and remove from menu) From 743232bf04d8ade18cfa2c791ed466ce48519878 Mon Sep 17 00:00:00 2001 From: Greg Pfeil Date: Thu, 30 Nov 2023 00:17:25 -0700 Subject: [PATCH 33/54] =?UTF-8?q?Don=E2=80=99t=20use=20`execvp`=20when=20w?= =?UTF-8?q?e=20know=20the=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 10 +++++++--- src/nix/run.hh | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 38482ed42..ae9be79a3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index c85eacded..396c93dbb 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, app.program, programArgs); + runProgramInStore(store, false, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index ea0a17897..d531f712d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system) @@ -58,7 +59,10 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + if (search) + execvp(program.c_str(), stringsToCharPtrs(args).data()); + else + execv(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); } @@ -132,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, *command.begin(), args); + runProgramInStore(store, true, *command.begin(), args); } }; @@ -194,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, app.program, allArgs); + runProgramInStore(store, false, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 97ddef19b..c62287e7e 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -6,6 +6,7 @@ namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system = std::nullopt); From f99e468640ab0eac7f07ac2b328222eb45dee8d8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Nov 2023 08:48:11 -0500 Subject: [PATCH 34/54] Avoid `/` in documentation URLs They are redundant and look weird. --- doc/manual/_redirects | 18 ++++++++++++++---- doc/manual/src/SUMMARY.md.in | 14 +++++++------- .../{advanced-topics.md => index.md} | 0 doc/manual/src/architecture/architecture.md | 2 +- .../command-ref/{command-ref.md => index.md} | 0 doc/manual/src/contributing/hacking.md | 2 +- .../contributing/{contributing.md => index.md} | 0 .../installation/{installation.md => index.md} | 0 .../{package-management.md => index.md} | 0 .../src/protocols/{protocols.md => index.md} | 0 doc/manual/src/quick-start.md | 2 +- .../{release-notes.md => index.md} | 0 12 files changed, 24 insertions(+), 14 deletions(-) rename doc/manual/src/advanced-topics/{advanced-topics.md => index.md} (100%) rename doc/manual/src/command-ref/{command-ref.md => index.md} (100%) rename doc/manual/src/contributing/{contributing.md => index.md} (100%) rename doc/manual/src/installation/{installation.md => index.md} (100%) rename doc/manual/src/package-management/{package-management.md => index.md} (100%) rename doc/manual/src/protocols/{protocols.md => index.md} (100%) rename doc/manual/src/release-notes/{release-notes.md => index.md} (100%) diff --git a/doc/manual/_redirects b/doc/manual/_redirects index 4ea289d86..2038671d7 100644 --- a/doc/manual/_redirects +++ b/doc/manual/_redirects @@ -13,18 +13,28 @@ # conventions: # - always force (!) since this allows re-using file names # - group related paths to ease readability -# - always append new redirects to the end of the file +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts # - redirects that should have been there but are missing can be inserted where they belong +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /contributing 301! + /expressions/expression-language /language/ 301! -/expressions/language-values /language/values 301! /expressions/language-constructs /language/constructs 301! /expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! /expressions/* /language/:splat 301! +/installation/installation /installation 301! + /package-management/basic-package-mgmt /command-ref/nix-env 301! - /package-management/channels* /command-ref/nix-channel 301! - +/package-management/package-management /package-management 301! /package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! +/protocols/protocols /protocols 301! + +/release-notes/release-notes /release-notes 301! diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8e7b4eeab..686d3e8d7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -31,11 +31,11 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) -- [Package Management](package-management/package-management.md) +- [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) +- [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) @@ -45,7 +45,7 @@ - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) @@ -102,18 +102,18 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [Protocols](protocols/protocols.md) +- [Protocols](protocols/index.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) +- [Contributing](contributing/index.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/release-notes.md) +- [Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md similarity index 100% rename from doc/manual/src/advanced-topics/advanced-topics.md rename to doc/manual/src/advanced-topics/index.md diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 79429508f..2fec4ed20 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects '---------------' ``` -At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 0a95334f7..9de5ad39b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -10,7 +10,7 @@ $ cd nix The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. -[installation instructions]: ../installation/installation.md +[installation instructions]: ../installation/index.md ## Building Nix with flakes diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md similarity index 100% rename from doc/manual/src/contributing/contributing.md rename to doc/manual/src/contributing/index.md diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md similarity index 100% rename from doc/manual/src/installation/installation.md rename to doc/manual/src/installation/index.md diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/protocols.md rename to doc/manual/src/protocols/index.md diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 1d2688ede..5f54abbde 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -13,7 +13,7 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/installation.md). + For other installation methods, see [here](installation/index.md). 1. See what installable packages are currently available in the channel: diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md similarity index 100% rename from doc/manual/src/release-notes/release-notes.md rename to doc/manual/src/release-notes/index.md From 44d21f6ef9783bed8812d39ff7b1a28a4883f84b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 20:36:50 +0100 Subject: [PATCH 35/54] keep generated documentation in a separate directory - helps navigating the code as it highlights which files are generated - makes it less error prone when working incrementally (although this should be just fixed by building out of tree) --- .gitignore | 2 +- .../manual/src/store/types/index.md.in | 4 ---- src/nix/local.mk | 19 ++++++++++++++++--- src/nix/main.cc | 2 +- src/nix/profile.md | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) rename src/nix/help-stores.md => doc/manual/src/store/types/index.md.in (99%) diff --git a/.gitignore b/.gitignore index 13a4dfb75..bcf9c4a01 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ perl/Makefile.config /src/nix/nix -/src/nix/doc +/src/nix/generated-doc # /src/nix-env/ /src/nix-env/nix-env diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in similarity index 99% rename from src/nix/help-stores.md rename to doc/manual/src/store/types/index.md.in index 47ba9b94d..bb166a1fc 100644 --- a/src/nix/help-stores.md +++ b/doc/manual/src/store/types/index.md.in @@ -1,5 +1,3 @@ -R"( - Nix supports different types of stores. These are described below. ## Store URL format @@ -42,5 +40,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. @stores@ - -)" diff --git a/src/nix/local.mk b/src/nix/local.mk index 57f8259c4..a21aa705f 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh +src/nix/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + src/nix/generated-doc/help-stores.md -src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md +src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) @cp $< $@ -src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh +src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh + +src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in + @mkdir -p $$(dirname $@) + @echo 'R"(' >> $@.tmp + @echo >> $@.tmp + @cat $^ >> $@.tmp + @echo >> $@.tmp + @echo ')"' >> $@.tmp + @mv $@.tmp $@ diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..49e637fb0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -297,7 +297,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "help-stores.md" + #include "generated-doc/help-stores.md" ; } diff --git a/src/nix/profile.md b/src/nix/profile.md index bd13f906f..9b2f86f4a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "doc/files/profiles.md.gen.hh" +#include "generated-doc/files/profiles.md.gen.hh" R""( From d5ffc94f336fc4032dd4009c14c148e390e10e16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:41:47 +0100 Subject: [PATCH 36/54] use lookup paths in helper expressions consistently this makes the files in question a bit more independent of source location. to find where the value is set and how it's wired up: rg nix=doc/manual --- doc/manual/generate-builtin-constants.nix | 2 +- doc/manual/generate-builtins.nix | 2 +- doc/manual/generate-manpage.nix | 30 +++++++++++++++---- doc/manual/generate-settings.nix | 2 +- doc/manual/generate-store-info.nix | 4 +-- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/generate-xp-features.nix | 2 +- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 8af80a02c..cccd1e279 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 813a287f5..05cae1c46 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 14136016d..c4b9d1335 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,9 +1,29 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs groupBy - concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; - showStoreDocs = import ./generate-store-info.nix; + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; in inlineHTML: commandDump: @@ -97,7 +117,7 @@ let ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} - ''; + ''; showOption = name: option: let result = trim '' diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 8736bb793..3add10075 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; - inherit (import ./utils.nix) concatStrings indent optionalString squash; + inherit (import ) concatStrings indent optionalString squash; in # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 36215aadf..73defcb71 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,7 +1,7 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ./utils.nix) concatStrings optionalString; - showSettings = import ./generate-settings.nix; + inherit (import ) concatStrings optionalString; + showSettings = import ; in inlineHTML: storesInfo: diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index 30e211c96..ec09f4b75 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index adb94355c..fc7c7d4cf 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: From cab41025d85d3b02f5175cf7ca2611c7a44c2cdd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:04:05 +0100 Subject: [PATCH 37/54] mention renaming of `nix doctor` --- doc/manual/rl-next/nix-config-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md index 08ad207cb..b2ad3c666 100644 --- a/doc/manual/rl-next/nix-config-show.md +++ b/doc/manual/rl-next/nix-config-show.md @@ -3,6 +3,6 @@ issues: #7672 prs: #9477 description: { -`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. +`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. } From 39de819edaef2dc3e308a490bbb2b1622a932771 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:08:22 +0200 Subject: [PATCH 38/54] rename debugging helper environment variable --- src/libutil/error.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8488e7e21..72c346cb5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -159,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test * git diff -U20 tests * */ -static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); /** * Print a position, if it is known. From 0301b8fc7354a94dc03b57f796bfa6e853758af8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:59:08 +0100 Subject: [PATCH 39/54] reword the experimental feature notice - put the highlight box around all the relevant instructions - simplify the wording - make the link more prominent by using the whole phrase for the link text --- doc/manual/generate-settings.nix | 21 ++++++------- doc/manual/generate-store-info.nix | 48 +++++++++++++++--------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 3add10075..74446b70b 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -31,18 +31,19 @@ let experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** + > > This setting is part of an > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${setting} = ... - ``` + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` ''; showDefault = documentDefault: defaultValue: diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 73defcb71..8e26edb65 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString; + inherit (import ) concatStrings optionalString squash; showSettings = import ; in @@ -10,36 +10,36 @@ let showStore = name: { settings, doc, experimentalFeature }: let + result = squash '' + # ${name} - result = '' - ## ${name} + ${doc} - ${doc} + ${experimentalFeatureNote} - ${experimentalFeatureNote} + ## Settings - ### Settings - - ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} - ''; + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; # markdown doesn't like spaces in URLs slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in result; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; in concatStrings (attrValues (mapAttrs showStore storesInfo)) From 908a011a4a2fe4e494e5b6e4c94f013f159f3616 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 00:50:20 +0100 Subject: [PATCH 40/54] Revert "Switch from std::regex to boost::regex" --- flake.nix | 22 +++++++--------------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 ++++++++++++----------------------- 3 files changed, 20 insertions(+), 39 deletions(-) diff --git a/flake.nix b/flake.nix index 33673575b..822b3d31e 100644 --- a/flake.nix +++ b/flake.nix @@ -157,7 +157,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost-nix}/lib" + "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -210,7 +210,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost-nix + boost lowdown-nix libsodium ] @@ -434,14 +434,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost-nix ]; + disallowedReferences = [ boost ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -451,9 +451,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done - install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -481,13 +481,9 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost-nix}/lib/libboost_context.dylib \ + -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib - install_name_tool \ - -change ${boost-nix}/lib/libboost_regex.dylib \ - $out/lib/libboost_regex.dylib \ - $out/lib/libnixexpr.dylib ''} ''; @@ -510,10 +506,6 @@ meta.mainProgram = "nix"; }); - boost-nix = final.boost.override { - enableIcu = false; - }; - lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index c07a18bb5..ed6bc761a 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread +libexpr_LDFLAGS += -lboost_context -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..146a7603c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -17,7 +17,6 @@ #include "primops.hh" #include -#include #include #include @@ -26,6 +25,7 @@ #include #include +#include #include #include @@ -3886,30 +3886,19 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); -// regex aliases, switch between boost and std -using regex = boost::regex; -using regex_error = boost::regex_error; -using cmatch = boost::cmatch; -using cregex_iterator = boost::cregex_iterator; -namespace regex_constants = boost::regex_constants; -// overloaded function alias -constexpr auto regex_match = [] (auto &&...args) { - return boost::regex_match(std::forward(args)...); - }; - struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - regex get(std::string_view re) + std::regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; + return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; } }; @@ -3929,8 +3918,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - cmatch match; - if (!regex_match(str.begin(), str.end(), match, regex)) { + std::cmatch match; + if (!std::regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3945,8 +3934,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -4009,8 +3998,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = cregex_iterator(str.begin(), str.end(), regex); - auto end = cregex_iterator(); + auto begin = std::cregex_iterator(str.begin(), str.end(), regex); + auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4049,8 +4038,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From 4781e7fa7048d2861172baaaa04a7be4b8a2b631 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:07:09 +0100 Subject: [PATCH 41/54] Document each store type on its own page This makes for more useful manual table of contents, that displays the information at a glance. The `nix help-stores` command is kept as-is, even though it will show up in the manual with the same information as these pages due to the way it is written as a "`--help`-style" command. Deciding what to do with that command is left for a later PR. This change also lists all store types at the top of the respective overview page. Co-authored-by: John Ericson `. + let + help-stores = '' + ${index} - maybeOptions = let - allVisibleOptions = filterAttrs - (_: o: ! o.hiddenCategory) - (details.flags // toplevel.flags); - in optionalString (allVisibleOptions != {}) '' - # Options + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" ] [ storesOverview ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + in + optionalString (details ? doc) ( + if match "@store-types@" details.doc != [ ] + then help-stores + else details.doc + ); - ${showOptions inlineHTML allVisibleOptions} + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options - > **Note** - > - > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. - ''; + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; showOptions = inlineHTML: allOptions: let diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 8e26edb65..57247a181 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,14 +1,20 @@ let - inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString squash; + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; showSettings = import ; in -inlineHTML: storesInfo: +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: let - showStore = name: { settings, doc, experimentalFeature }: + showStore = { name, slug }: { settings, doc, experimentalFeature }: let result = squash '' # ${name} @@ -22,9 +28,6 @@ let ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} ''; - # markdown doesn't like spaces in URLs - slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** > @@ -42,4 +45,13 @@ let ''; in result; -in concatStrings (attrValues (mapAttrs showStore storesInfo)) + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); + +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000..3b78a0e1b --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index fc7c7d4cf..0eec0e1da 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -8,4 +8,6 @@ let ${doc} ''; -in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fa9db9f02..456000d3d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 686d3e8d7..c67ddc6cb 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,8 @@ - [File System Object](store/file-system-object.md) - [Store Object](store/store-object.md) - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..f7e24d96b 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -426,7 +426,7 @@ This leads to the following guidelines: ### Examples -This is bad, because all keys must be assumed to be store implementations: +This is bad, because all keys must be assumed to be store types: ```json { diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 8a5305062..f1e8f1402 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -2,4 +2,4 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index bb166a1fc..b4db553a2 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -1,4 +1,6 @@ -Nix supports different types of stores. These are described below. +Nix supports different types of stores: + +@store-types@ ## Store URL format @@ -39,4 +41,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. -@stores@ diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 849832b2c..19ff49b64 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,5 +1,11 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); @@ -18,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 401acc38e..193972272 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs() .longName = "eval-store", .description = R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. )", .category = category, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..4ecdda55c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv() .doc = R"( Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. - This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): ```shell-session $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 838d2aba2..38b0d516c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -117,10 +117,11 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) - for supported store types and settings. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. )"}; Setting keepFailed{this, false, "keep-failed", @@ -759,7 +760,7 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. @@ -778,7 +779,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f16949f42..3d3919882 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -20,7 +20,7 @@ namespace nix { -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index bf55d20cf..4ce4ffc4c 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig const Setting remoteStore{this, "", "remote-store", R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )"}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8b6bf9aed..e28baf34e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -30,7 +30,7 @@ namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev * - ‘ssh://[user@]’: A remote Nix store accessed by running * ‘nix-store --serve’ via SSH. * - * You can pass parameters to the store implementation by appending + * You can pass parameters to the store type by appending * ‘?key=value&key=value&...’ to the URI. */ ref openStore(const std::string & uri = settings.storeUri.get(), diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 2418e3f4c..e4bdb8cb3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -252,7 +252,7 @@ constexpr std::array xpFeatureDetails .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", }, { diff --git a/src/nix/nix.md b/src/nix/nix.md index eb150f03b..749456014 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -235,10 +235,14 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. These are documented -in [`nix help-stores`](./nix3-help-stores.md). +Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. -# Shebang interpreter +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. From 333ea684b065318aa49aec367c995b3d8c5d65ed Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 01:39:52 +0100 Subject: [PATCH 42/54] Add boost::regex regression test --- src/libexpr/tests/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index d820b860e..7485fa0d0 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -814,6 +814,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); From ea2dd166235e049699cf7f70c243c2b83089f824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Dec 2023 15:35:21 +0100 Subject: [PATCH 43/54] Use a proper enum rather than a boolean in runProgramInStore Makes the call-site much easier to understand. --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 8 ++++---- src/nix/run.hh | 7 ++++++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ae9be79a3..606b044b0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 396c93dbb..059904150 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, false, app.program, programArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index d531f712d..efc0c56a1 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system) @@ -59,7 +59,7 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - if (search) + if (useSearchPath == UseSearchPath::Use) execvp(program.c_str(), stringsToCharPtrs(args).data()); else execv(program.c_str(), stringsToCharPtrs(args).data()); @@ -136,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, true, *command.begin(), args); + runProgramInStore(store, UseSearchPath::Use, *command.begin(), args); } }; @@ -198,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, false, app.program, allArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index c62287e7e..a55917b06 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -5,8 +5,13 @@ namespace nix { +enum struct UseSearchPath { + Use, + DontUse +}; + void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system = std::nullopt); From d59bdbe4fd757d99b6625db1d3560a39a371d9e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 10:20:19 -0500 Subject: [PATCH 44/54] Add two missing `#include "nar-info.hh"` GitHub's racy CI caused this oversight to sneak through. --- src/libstore/tests/nar-info.cc | 1 + src/nix/path-info.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc index c5b21d56b..88e6e1add 100644 --- a/src/libstore/tests/nar-info.cc +++ b/src/libstore/tests/nar-info.cc @@ -2,6 +2,7 @@ #include #include "path-info.hh" +#include "nar-info.hh" #include "tests/characterization.hh" #include "tests/libstore.hh" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 080d6bbf1..5f10cfb61 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "common-args.hh" +#include "nar-info.hh" #include #include From 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 10:20:28 -0400 Subject: [PATCH 45/54] Move tests to separate directories, and document Today, with the tests inside a `tests` intermingled with the corresponding library's source code, we have a few problems: - We have to be careful that wildcards don't end up with tests being built as part of Nix proper, or test headers being installed as part of Nix proper. - Tests in libraries but not executables is not right: - It means each executable runs the previous unit tests again, because it needs the libraries. - It doesn't work right on Windows, which doesn't want you to load a DLL just for the side global variable . It could be made to work with the dlopen equivalent, but that's gross! This reorg solves these problems. There is a remaining problem which is that sibbling headers (like `hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end up shadowing each other. This PR doesn't solve that. That is left as future work for a future PR. Co-authored-by: Valentin Gagarin --- .gitignore | 6 +- Makefile | 10 ++- doc/internal-api/doxygen.cfg.in | 12 ++- doc/manual/src/contributing/testing.md | 67 ++++++++++---- flake.nix | 2 +- mk/common-test.sh | 2 +- mk/programs.mk | 2 +- src/libexpr/tests/local.mk | 23 ----- src/libstore/tests/local.mk | 37 -------- src/libutil/tests/local.mk | 41 --------- tests/unit/libexpr-support/local.mk | 23 +++++ .../unit/libexpr-support}/tests/libexpr.hh | 0 .../libexpr-support/tests/value/context.cc | 30 +++++++ .../libexpr-support}/tests/value/context.hh | 2 +- .../unit/libexpr}/derived-path.cc | 0 .../unit/libexpr}/error_traces.cc | 0 .../tests => tests/unit/libexpr}/flakeref.cc | 0 .../tests => tests/unit/libexpr}/json.cc | 0 tests/unit/libexpr/local.mk | 36 ++++++++ .../tests => tests/unit/libexpr}/primops.cc | 0 .../unit/libexpr}/search-path.cc | 0 .../tests => tests/unit/libexpr}/trivial.cc | 0 .../unit/libexpr}/value/context.cc | 30 ------- .../unit/libexpr}/value/print.cc | 0 tests/unit/libstore-support/local.mk | 21 +++++ .../libstore-support/tests/derived-path.cc | 57 ++++++++++++ .../libstore-support}/tests/derived-path.hh | 0 .../unit/libstore-support}/tests/libstore.hh | 0 .../libstore-support/tests/outputs-spec.cc | 24 +++++ .../libstore-support}/tests/outputs-spec.hh | 2 +- tests/unit/libstore-support/tests/path.cc | 82 ++++++++++++++++++ .../unit/libstore-support}/tests/path.hh | 3 + .../unit/libstore-support}/tests/protocol.hh | 2 +- .../unit/libstore}/common-protocol.cc | 0 .../data}/common-protocol/content-address.bin | Bin .../data}/common-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../common-protocol/optional-store-path.bin | Bin .../data}/common-protocol/realisation.bin | Bin .../libstore/data}/common-protocol/set.bin | Bin .../data}/common-protocol/store-path.bin | Bin .../libstore/data}/common-protocol/string.bin | Bin .../libstore/data}/common-protocol/vector.bin | Bin .../derivation/bad-old-version-dyn-deps.drv | 0 .../libstore/data}/derivation/bad-version.drv | 0 .../data}/derivation/dynDerivationDeps.drv | 0 .../data}/derivation/dynDerivationDeps.json | 0 .../data}/derivation/output-caFixedFlat.json | 0 .../data}/derivation/output-caFixedNAR.json | 0 .../data}/derivation/output-caFixedText.json | 0 .../data}/derivation/output-caFloating.json | 0 .../data}/derivation/output-deferred.json | 0 .../data}/derivation/output-impure.json | 0 .../derivation/output-inputAddressed.json | 0 .../unit/libstore/data}/derivation/simple.drv | 0 .../libstore/data}/derivation/simple.json | 0 .../unit/libstore/data}/nar-info/impure.json | 0 .../unit/libstore/data}/nar-info/pure.json | 0 .../unit/libstore/data}/path-info/impure.json | 0 .../unit/libstore/data}/path-info/pure.json | 0 .../data}/serve-protocol/build-result-2.2.bin | Bin .../data}/serve-protocol/build-result-2.3.bin | Bin .../data}/serve-protocol/build-result-2.6.bin | Bin .../data}/serve-protocol/content-address.bin | Bin .../data}/serve-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../serve-protocol/optional-store-path.bin | Bin .../data}/serve-protocol/realisation.bin | Bin .../libstore/data}/serve-protocol/set.bin | Bin .../data}/serve-protocol/store-path.bin | Bin .../libstore/data}/serve-protocol/string.bin | Bin .../libstore/data}/serve-protocol/vector.bin | Bin .../worker-protocol/build-result-1.27.bin | Bin .../worker-protocol/build-result-1.28.bin | Bin .../worker-protocol/build-result-1.29.bin | Bin .../data}/worker-protocol/content-address.bin | Bin .../worker-protocol/derived-path-1.29.bin | Bin .../worker-protocol/derived-path-1.30.bin | Bin .../data}/worker-protocol/drv-output.bin | Bin .../keyed-build-result-1.29.bin | Bin .../optional-content-address.bin | Bin .../worker-protocol/optional-store-path.bin | Bin .../worker-protocol/optional-trusted-flag.bin | Bin .../data}/worker-protocol/realisation.bin | Bin .../libstore/data}/worker-protocol/set.bin | Bin .../data}/worker-protocol/store-path.bin | Bin .../libstore/data}/worker-protocol/string.bin | Bin .../unkeyed-valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.16.bin | Bin .../libstore/data}/worker-protocol/vector.bin | Bin .../unit/libstore}/derivation.cc | 2 +- .../unit/libstore}/derived-path.cc | 53 ----------- .../unit/libstore}/downstream-placeholder.cc | 0 tests/unit/libstore/local.mk | 31 +++++++ .../tests => tests/unit/libstore}/machines.cc | 4 +- .../unit/libstore}/nar-info-disk-cache.cc | 0 .../tests => tests/unit/libstore}/nar-info.cc | 2 +- .../unit/libstore}/outputs-spec.cc | 27 +----- .../unit/libstore}/path-info.cc | 2 +- .../tests => tests/unit/libstore}/path.cc | 73 ---------------- .../unit/libstore}/references.cc | 0 .../unit/libstore}/serve-protocol.cc | 0 .../libstore}/test-data/machines.bad_format | 0 .../unit/libstore}/test-data/machines.valid | 0 .../unit/libstore}/worker-protocol.cc | 0 tests/unit/libutil-support/local.mk | 19 ++++ .../tests/characterization.hh | 4 +- tests/unit/libutil-support/tests/hash.cc | 20 +++++ .../unit/libutil-support}/tests/hash.hh | 0 .../tests => tests/unit/libutil}/args.cc | 6 +- .../unit/libutil}/canon-path.cc | 0 .../unit/libutil}/chunked-vector.cc | 0 .../tests => tests/unit/libutil}/closure.cc | 0 .../unit/libutil}/compression.cc | 0 .../tests => tests/unit/libutil}/config.cc | 0 .../unit/libutil/data}/git/check-data.sh | 2 +- .../libutil/data}/git/hello-world-blob.bin | Bin .../unit/libutil/data}/git/hello-world.bin | Bin .../unit/libutil/data}/git/tree.bin | Bin .../unit/libutil/data}/git/tree.txt | 0 .../tests => tests/unit/libutil}/git.cc | 6 +- .../tests => tests/unit/libutil}/hash.cc | 20 +---- .../tests => tests/unit/libutil}/hilite.cc | 0 .../unit/libutil}/json-utils.cc | 0 tests/unit/libutil/local.mk | 31 +++++++ .../tests => tests/unit/libutil}/logging.cc | 0 .../tests => tests/unit/libutil}/lru-cache.cc | 0 .../tests => tests/unit/libutil}/pool.cc | 0 .../unit/libutil}/references.cc | 0 .../unit/libutil}/suggestions.cc | 0 .../tests => tests/unit/libutil}/tests.cc | 0 .../tests => tests/unit/libutil}/url.cc | 0 .../unit/libutil}/xml-writer.cc | 0 134 files changed, 464 insertions(+), 352 deletions(-) delete mode 100644 src/libexpr/tests/local.mk delete mode 100644 src/libstore/tests/local.mk delete mode 100644 src/libutil/tests/local.mk create mode 100644 tests/unit/libexpr-support/local.mk rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%) create mode 100644 tests/unit/libexpr-support/tests/value/context.cc rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%) rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%) create mode 100644 tests/unit/libexpr/local.mk rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%) rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%) create mode 100644 tests/unit/libstore-support/local.mk create mode 100644 tests/unit/libstore-support/tests/derived-path.cc rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%) rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%) create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%) create mode 100644 tests/unit/libstore-support/tests/path.cc rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%) rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%) rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%) rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%) rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%) rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%) create mode 100644 tests/unit/libstore/local.mk rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%) rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%) rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%) rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%) rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%) create mode 100644 tests/unit/libutil-support/local.mk rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%) create mode 100644 tests/unit/libutil-support/tests/hash.cc rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%) rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%) rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%) rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%) rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%) rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/json-utils.cc (100%) create mode 100644 tests/unit/libutil/local.mk rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%) diff --git a/.gitignore b/.gitignore index 767a5d6ed..d9f9d949b 100644 --- a/.gitignore +++ b/.gitignore @@ -45,14 +45,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix diff --git a/Makefile b/Makefile index 92727bea5..eea297c89 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,13 @@ makefiles = \ endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) -UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk endif ifeq ($(ENABLE_TESTS), yes) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 599be2470..ad5af97e6 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,17 +39,21 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 0b45b88a3..d8d162379 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. [googletest]: https://google.github.io/googletest/ [rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing ### Source and header layout @@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` > src > ├── libexpr +> │ ├── local.mk > │ ├── value/context.hh > │ ├── value/context.cc +> │ … +> │ +> ├── tests > │ │ > │ … -> └── tests -> │ ├── value/context.hh -> │ ├── value/context.cc +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … > │ │ -> │ … -> │ -> ├── unit-test-data -> │ ├── libstore -> │ │ ├── worker-protocol/content-address.bin -> │ │ … -> │ … +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … > … > ``` -The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. -The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. -The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. -> **Note** -> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. -> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. -> That is why we have the test data nested within a single `unit-test-data` directory. +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. ### Running tests @@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. diff --git a/flake.nix b/flake.nix index 822b3d31e..e2e510cbc 100644 --- a/flake.nix +++ b/flake.nix @@ -93,7 +93,7 @@ ./misc ./precompiled-headers.h ./src - ./unit-test-data + ./tests/unit ./COPYING ./scripts/local.mk functionalTestFiles diff --git a/mk/common-test.sh b/mk/common-test.sh index 00ccd1584..2783d293b 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,7 +1,7 @@ # Remove overall test dir (at most one of the two should match) and # remove file extension. test_name=$(echo -n "$test" | sed \ - -e "s|^unit-test-data/||" \ + -e "s|^tests/unit/[^/]*/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) diff --git a/mk/programs.mk b/mk/programs.mk index a88d9d949..6235311e9 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 7689a03e0..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index e9b8b4f99..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests-exe_INSTALL_DIR := $(checkbindir) -else - libstore-tests-exe_INSTALL_DIR := -endif - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checklibdir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 66886c45f..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,41 +0,0 @@ -check: libutil-tests-exe_RUN - -programs += libutil-tests-exe - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests-exe_INSTALL_DIR := $(checkbindir) -else - libutil-tests-exe_INSTALL_DIR := -endif - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checklibdir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -check: unit-test-data/libutil/git/check-data.sh.test - -$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..28e87b8f2 --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 100% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3..8c68c78bb 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 100% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 100% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..5743880d7 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 100% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889ab..761286dbd 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 100% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..d5d657c91 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b33..f5bf9042d 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..e5f169e94 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310..4751b3373 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh similarity index 96% rename from src/libstore/tests/protocol.hh rename to tests/unit/libstore-support/tests/protocol.hh index 466032a79..3c9e52c11 100644 --- a/src/libstore/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,7 +12,7 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + Path unitTestData = getUnitTestData() + "/" + protocolDir; Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc similarity index 100% rename from src/libstore/tests/common-protocol.cc rename to tests/unit/libstore/common-protocol.cc diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/content-address.bin rename to tests/unit/libstore/data/common-protocol/content-address.bin diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/drv-output.bin rename to tests/unit/libstore/data/common-protocol/drv-output.bin diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-content-address.bin rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-store-path.bin rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/realisation.bin rename to tests/unit/libstore/data/common-protocol/realisation.bin diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/set.bin rename to tests/unit/libstore/data/common-protocol/set.bin diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/store-path.bin rename to tests/unit/libstore/data/common-protocol/store-path.bin diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/string.bin rename to tests/unit/libstore/data/common-protocol/string.bin diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/vector.bin rename to tests/unit/libstore/data/common-protocol/vector.bin diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-version.drv rename to tests/unit/libstore/data/derivation/bad-version.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.json rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedFlat.json rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedNAR.json rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedText.json rename to tests/unit/libstore/data/derivation/output-caFixedText.json diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFloating.json rename to tests/unit/libstore/data/derivation/output-caFloating.json diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json similarity index 100% rename from unit-test-data/libstore/derivation/output-deferred.json rename to tests/unit/libstore/data/derivation/output-deferred.json diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json similarity index 100% rename from unit-test-data/libstore/derivation/output-impure.json rename to tests/unit/libstore/data/derivation/output-impure.json diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json similarity index 100% rename from unit-test-data/libstore/derivation/output-inputAddressed.json rename to tests/unit/libstore/data/derivation/output-inputAddressed.json diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv similarity index 100% rename from unit-test-data/libstore/derivation/simple.drv rename to tests/unit/libstore/data/derivation/simple.drv diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json similarity index 100% rename from unit-test-data/libstore/derivation/simple.json rename to tests/unit/libstore/data/derivation/simple.json diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json similarity index 100% rename from unit-test-data/libstore/nar-info/impure.json rename to tests/unit/libstore/data/nar-info/impure.json diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json similarity index 100% rename from unit-test-data/libstore/nar-info/pure.json rename to tests/unit/libstore/data/nar-info/pure.json diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json similarity index 100% rename from unit-test-data/libstore/path-info/impure.json rename to tests/unit/libstore/data/path-info/impure.json diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json similarity index 100% rename from unit-test-data/libstore/path-info/pure.json rename to tests/unit/libstore/data/path-info/pure.json diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/content-address.bin rename to tests/unit/libstore/data/serve-protocol/content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/drv-output.bin rename to tests/unit/libstore/data/serve-protocol/drv-output.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/realisation.bin rename to tests/unit/libstore/data/serve-protocol/realisation.bin diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/set.bin rename to tests/unit/libstore/data/serve-protocol/set.bin diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/store-path.bin rename to tests/unit/libstore/data/serve-protocol/store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/string.bin rename to tests/unit/libstore/data/serve-protocol/string.bin diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/vector.bin rename to tests/unit/libstore/data/serve-protocol/vector.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/content-address.bin rename to tests/unit/libstore/data/worker-protocol/content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/drv-output.bin rename to tests/unit/libstore/data/worker-protocol/drv-output.bin diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/realisation.bin rename to tests/unit/libstore/data/worker-protocol/realisation.bin diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/set.bin rename to tests/unit/libstore/data/worker-protocol/set.bin diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/store-path.bin rename to tests/unit/libstore/data/worker-protocol/store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/string.bin rename to tests/unit/libstore/data/worker-protocol/string.bin diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/vector.bin rename to tests/unit/libstore/data/worker-protocol/vector.bin diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc similarity index 99% rename from src/libstore/tests/derivation.cc rename to tests/unit/libstore/derivation.cc index 7becfa5ab..a7f4488fa 100644 --- a/src/libstore/tests/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,7 +13,7 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/derivation"; + Path unitTestData = getUnitTestData() + "/derivation"; public: Path goldenMaster(std::string_view testStem) const override { diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c0801..c62d79a78 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..63f6d011f --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index fede328ea..5b66e5a5b 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc similarity index 98% rename from src/libstore/tests/nar-info.cc rename to tests/unit/libstore/nar-info.cc index 88e6e1add..4f124e89e 100644 --- a/src/libstore/tests/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -13,7 +13,7 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + Path unitTestData = getUnitTestData() + "/nar-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 952945185..456196be1 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc similarity index 97% rename from src/libstore/tests/path-info.cc rename to tests/unit/libstore/path-info.cc index 49bf623bd..18f00ca19 100644 --- a/src/libstore/tests/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,7 +12,7 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/path-info"; + Path unitTestData = getUnitTestData() + "/path-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 59% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index 5a84d646c..30631b5fd 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc similarity index 100% rename from src/libstore/tests/serve-protocol.cc rename to tests/unit/libstore/serve-protocol.cc diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc similarity index 100% rename from src/libstore/tests/worker-protocol.cc rename to tests/unit/libstore/worker-protocol.cc diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..43a1551e5 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh similarity index 95% rename from src/libutil/tests/characterization.hh rename to tests/unit/libutil-support/tests/characterization.hh index 6eb513d68..9d6c850f0 100644 --- a/src/libutil/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -9,8 +9,8 @@ namespace nix { /** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. */ static Path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..577e9890e --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc similarity index 98% rename from src/libutil/tests/args.cc rename to tests/unit/libutil/args.cc index bea74a8c8..950224430 100644 --- a/src/libutil/tests/args.cc +++ b/tests/unit/libutil/args.cc @@ -1,5 +1,5 @@ -#include "../args.hh" -#include "libutil/fs-sink.hh" +#include "args.hh" +#include "fs-sink.hh" #include #include @@ -165,4 +165,4 @@ RC_GTEST_PROP( #endif -} \ No newline at end of file +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh similarity index 98% rename from unit-test-data/libutil/git/check-data.sh rename to tests/unit/libutil/data/git/check-data.sh index 68b705c95..b3f59c4f1 100644 --- a/unit-test-data/libutil/git/check-data.sh +++ b/tests/unit/libutil/data/git/check-data.sh @@ -2,7 +2,7 @@ set -eu -o pipefail -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data mkdir -p $TEST_ROOT repo="$TEST_ROOT/scratch" diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world-blob.bin rename to tests/unit/libutil/data/git/hello-world-blob.bin diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world.bin rename to tests/unit/libutil/data/git/hello-world.bin diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin similarity index 100% rename from unit-test-data/libutil/git/tree.bin rename to tests/unit/libutil/data/git/tree.bin diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt similarity index 100% rename from unit-test-data/libutil/git/tree.txt rename to tests/unit/libutil/data/git/tree.txt diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc similarity index 97% rename from src/libutil/tests/git.cc rename to tests/unit/libutil/git.cc index 2842ea4d0..551a2d105 100644 --- a/src/libutil/tests/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,7 +11,7 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/libutil/git"; + Path unitTestData = getUnitTestData() + "/git"; public: @@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) { /** * This data is for "shallow" tree tests. However, we use "real" hashes - * so that we can check our test data in the corresponding functional - * test (`git-hashing/unit-test-data`). + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). */ const static Tree tree = { { diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 92% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 9a5ebbb30..92291afce 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -68,7 +64,6 @@ namespace nix { "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; @@ -95,16 +90,3 @@ namespace nix { ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); } } - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} - -} diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/src/libutil/tests/json-utils.cc b/tests/unit/libutil/json-utils.cc similarity index 100% rename from src/libutil/tests/json-utils.cc rename to tests/unit/libutil/json-utils.cc diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..930efb90b --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 100% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 100% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 100% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc From 59c4c82aebb814d864548c3ad2e9128ab6e902bf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 00:56:23 +0100 Subject: [PATCH 46/54] fix links in stores overview --- doc/manual/generate-manpage.nix | 3 ++- doc/manual/src/store/types/index.md.in | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 365422af7..ae31b2a1f 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -103,7 +103,8 @@ let ${allStores} ''; index = replaceStrings - [ "@store-types@" ] [ storesOverview ] + [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ] + [ storesOverview "#local-store" "#local-daemon-store" ] details.doc; storesOverview = let diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index b4db553a2..a35161ce8 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -29,15 +29,15 @@ supported settings for each store type are documented below. The special store URL `auto` causes Nix to automatically select a store as follows: -* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` +* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix` is writable by the current user. * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect - to the Nix daemon listening on that socket](#local-daemon-store). + to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](#local-store) +* Otherwise, on Linux only, use the [local chroot store](./local-store.md) `~/.local/share/nix/root`, which will be created automatically if it does not exist. -* Otherwise, use the [local store](#local-store) `/nix/store`. +* Otherwise, use the [local store](./local-store.md) `/nix/store`. From 24b781773f3d24b62b1c36154c36fc98417cbcdb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 03:02:58 +0100 Subject: [PATCH 47/54] fix random docs errors remove link to the contributing guide from user documentation. it doesn't help here, and the target at first glance shows redundant information. --- src/libexpr/primops.cc | 2 +- src/libstore/globals.hh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 63c90795a..c2499bdae 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4400,7 +4400,7 @@ void EvalState::createBaseEnv() addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..8da9e371f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -201,7 +201,7 @@ public: Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. - The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` - `x86_64-darwin` @@ -761,7 +761,7 @@ public: "substituters", R"( A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. - A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. Lower value means higher priority. From 2e5abc0fd0d5d45e125e1d981958149624268090 Mon Sep 17 00:00:00 2001 From: wh0 Date: Sun, 3 Dec 2023 17:18:58 -0800 Subject: [PATCH 48/54] tests: avoid a chroot store without sandbox support --- tests/functional/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index fad1def59..3d4a4b097 100644 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override" restartDaemon +requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR From 3c310bde2e492c2dd8bdccdfd80076231905a429 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:40:56 +0100 Subject: [PATCH 49/54] reword description for the `fetch-tree` experimental feature without knowing a lot of context, it's not clear who "we" are in that text. I'm also strongly opposed to adding procedural notes into a reference manual; it just won't age well. this change leaves a factual description of the experimental feature and its purpose. --- src/libutil/experimental-features.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e4bdb8cb3..9b46fc5b0 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,12 +80,11 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. - This built-in was previously guarded by the `flakes` experimental feature because of that overlap, - but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. - Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. )", }, { From 5fe2accb754249df6cb8f840330abfcf3bd26695 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:44:34 +0100 Subject: [PATCH 50/54] fix up release note --- doc/manual/src/release-notes/rl-2.19.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index 4eecaf929..ba6eb9c64 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -18,7 +18,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: From 823512c1e705d1fce8dfb8cde65228364c9a8045 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:10 +0000 Subject: [PATCH 51/54] Bump zeebe-io/backport-action from 2.1.1 to 2.2.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.1 to 2.2.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.1...v2.2.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 975c90b91..85ddcfad3 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.1.1 + uses: zeebe-io/backport-action@v2.2.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From c446e5294dbc12729e7bc55ee10b40dbaeeaacf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:14 +0000 Subject: [PATCH 52/54] Bump cachix/install-nix-action from 23 to 24 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 23 to 24. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v23...v24) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..34a23b5f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,7 +114,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From e6a3cbfceb66e06184b625a3913a786f68e71a1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:18 +0000 Subject: [PATCH 53/54] Bump cachix/cachix-action from 12 to 13 Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 12 to 13. - [Release notes](https://github.com/cachix/cachix-action/releases) - [Commits](https://github.com/cachix/cachix-action/compare/v12...v13) --- updated-dependencies: - dependency-name: cachix/cachix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..033832c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -65,7 +65,7 @@ jobs: - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -119,7 +119,7 @@ jobs: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' From e488a43f457f3ef9dba92184428bbe5381fe2634 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:25 +0000 Subject: [PATCH 54/54] Bump actions/labeler from 4 to 5 Bumps [actions/labeler](https://github.com/actions/labeler) from 4 to 5. - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/labeler dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index d83cb4f18..34aa4e6bd 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'NixOS' steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: false